Archive cores dumped during test, record test as failed if any seen.

diff --git a/teuthology/misc.py b/teuthology/misc.py
index 5b8fc26472f96f48704e3f8271d3ea16461299f0..25f77726ada8d3e4135bd043900c055578fd8c9e 100644 (file)
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -169,6 +169,7 @@ def create_simple_monmap(remote, conf):
     log.debug('Ceph mon addresses: %s', addresses)
 
     args = [
+        '/tmp/cephtest/enable-coredump',
         '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
         '/tmp/cephtest/archive/coverage',
         '/tmp/cephtest/binary/usr/local/bin/monmaptool',
@@ -216,6 +217,7 @@ def wait_until_healthy(remote):
     while True:
         r = remote.run(
             args=[
+                '/tmp/cephtest/enable-coredump',
                 '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                 '/tmp/cephtest/archive/coverage',
                 '/tmp/cephtest/binary/usr/local/bin/ceph',
@@ -258,6 +260,7 @@ def wait_until_fuse_mounted(remote, fuse, mountpoint):
 def write_secret_file(remote, role, filename):
     remote.run(
         args=[
+            '/tmp/cephtest/enable-coredump',
             '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
             '/tmp/cephtest/archive/coverage',
             '/tmp/cephtest/binary/usr/local/bin/cauthtool',

diff --git a/teuthology/run.py b/teuthology/run.py
index 11b5fd7240ef0c7d93e07fa75bc24e035e25810f..c27e4a9b053d3cc2164217c876e9655e99bc8e26 100644 (file)
--- a/teuthology/run.py
+++ b/teuthology/run.py
@@ -96,6 +96,7 @@ def main():
         {'internal.check_conflict': None},
         {'internal.base': None},
         {'internal.archive': None},
+        {'internal.coredump': None},
         ]
 
     from teuthology.run_tasks import run_tasks

diff --git a/teuthology/task/ceph.py b/teuthology/task/ceph.py
index 8b7bef86aa3d7ff5b40dd3364d53b2e0e6bbd0d3..d3ffc7bdb813ebb7ca956b3dd4e45cbbefb99140 100644 (file)
--- a/teuthology/task/ceph.py
+++ b/teuthology/task/ceph.py
@@ -58,7 +58,7 @@ def ceph_log(ctx, config):
 @contextlib.contextmanager
 def ship_utilities(ctx, config):
     assert config is None
-    FILES = ['daemon-helper']
+    FILES = ['daemon-helper', 'enable-coredump']
     for filename in FILES:
         log.info('Shipping %r...', filename)
         src = os.path.join(os.path.dirname(__file__), filename)
@@ -185,6 +185,7 @@ def cluster(ctx, config):
     log.info('Setting up mon.0...')
     ctx.cluster.only('mon.0').run(
         args=[
+            '/tmp/cephtest/enable-coredump',
             '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
             coverage_dir,
             '/tmp/cephtest/binary/usr/local/bin/cauthtool',
@@ -194,6 +195,7 @@ def cluster(ctx, config):
         )
     ctx.cluster.only('mon.0').run(
         args=[
+            '/tmp/cephtest/enable-coredump',
             '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
             coverage_dir,
             '/tmp/cephtest/binary/usr/local/bin/cauthtool',
@@ -211,6 +213,7 @@ def cluster(ctx, config):
     log.info('Creating admin key on mon.0...')
     ctx.cluster.only('mon.0').run(
         args=[
+            '/tmp/cephtest/enable-coredump',
             '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
             coverage_dir,
             '/tmp/cephtest/binary/usr/local/bin/cauthtool',
@@ -254,6 +257,7 @@ def cluster(ctx, config):
     run.wait(
         mons.run(
             args=[
+                '/tmp/cephtest/enable-coredump',
                 '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                 coverage_dir,
                 '/tmp/cephtest/binary/usr/local/bin/osdmaptool',
@@ -275,6 +279,7 @@ def cluster(ctx, config):
         for id_ in teuthology.roles_of_type(roles_for_host, 'osd'):
             remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/binary/usr/local/bin/cauthtool',
@@ -291,6 +296,7 @@ def cluster(ctx, config):
         for id_ in teuthology.roles_of_type(roles_for_host, 'mds'):
             remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/binary/usr/local/bin/cauthtool',
@@ -307,6 +313,7 @@ def cluster(ctx, config):
         for id_ in teuthology.roles_of_type(roles_for_host, 'client'):
             remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/binary/usr/local/bin/cauthtool',
@@ -351,6 +358,7 @@ def cluster(ctx, config):
         run.wait(
             mons.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/binary/usr/local/bin/cauthtool',
@@ -369,6 +377,7 @@ def cluster(ctx, config):
         for id_ in teuthology.roles_of_type(roles_for_host, 'mon'):
             remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/binary/usr/local/bin/cmon',
@@ -392,6 +401,7 @@ def cluster(ctx, config):
                 )
             remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/binary/usr/local/bin/cosd',
@@ -448,6 +458,7 @@ def mon(ctx, config):
         for id_ in teuthology.roles_of_type(roles_for_host, 'mon'):
             proc = remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/daemon-helper',
@@ -489,6 +500,7 @@ def osd(ctx, config):
         for id_ in teuthology.roles_of_type(roles_for_host, 'osd'):
             proc = remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/daemon-helper',
@@ -530,6 +542,7 @@ def mds(ctx, config):
         for id_ in teuthology.roles_of_type(roles_for_host, 'mds'):
             proc = remote.run(
                 args=[
+                    '/tmp/cephtest/enable-coredump',
                     '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                     coverage_dir,
                     '/tmp/cephtest/daemon-helper',

diff --git a/teuthology/task/cfuse.py b/teuthology/task/cfuse.py
index 23e2636903392b1c33687904ca2a43c4b1b25c18..988fca18da6638f6fc0457f391388c9ceaaec4c7 100644 (file)
--- a/teuthology/task/cfuse.py
+++ b/teuthology/task/cfuse.py
@@ -63,6 +63,7 @@ def task(ctx, config):
             )
         proc = remote.run(
             args=[
+                '/tmp/cephtest/enable-coredump',
                 '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                 '/tmp/cephtest/archive/coverage',
                 '/tmp/cephtest/daemon-helper',

diff --git a/teuthology/task/enable-coredump b/teuthology/task/enable-coredump
new file mode 100755 (executable)
index 0000000..4257764
--- /dev/null
+++ b/teuthology/task/enable-coredump
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+ulimit -c unlimited
+exec "$@"

diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py
index 937f1d4c1b69deecd637bb30c747057bd6a6d619..7722ea63eba87b8fdb2a30dd9644e67b2183ba58 100644 (file)
--- a/teuthology/task/internal.py
+++ b/teuthology/task/internal.py
@@ -1,3 +1,4 @@
+from cStringIO import StringIO
 import contextlib
 import gevent
 import logging
@@ -134,3 +135,52 @@ def archive(ctx, config):
                     wait=False,
                     ),
                 )
+
+@contextlib.contextmanager
+def coredump(ctx, config):
+    log.info('Enabling coredump saving...')
+    run.wait(
+        ctx.cluster.run(
+            args=[
+                'install', '-d', '-m0755', '--',
+                '/tmp/cephtest/archive/coredump',
+                run.Raw('&&'),
+                'sudo', 'sysctl', '-w', 'kernel.core_pattern=/tmp/cephtest/archive/coredump/%t.%p.core',
+                ],
+            wait=False,
+            )
+        )
+
+    try:
+        yield
+    finally:
+        run.wait(
+            ctx.cluster.run(
+                args=[
+                    'sudo', 'sysctl', '-w', 'kernel.core_pattern=core',
+                    run.Raw('&&'),
+                    # don't litter the archive dir if there were no cores dumped
+                    'rmdir',
+                    '--ignore-fail-on-non-empty',
+                    '--',
+                    '/tmp/cephtest/archive/coredump',
+                    ],
+                wait=False,
+                )
+            )
+
+        # set success=false if the dir is still there = coredumps were
+        # seen
+        processes = ctx.cluster.run(
+            args=[
+                'if', 'test', '!', '-e', '/tmp/cephtest/archive/coredump', run.Raw(';'), 'then',
+                'echo', 'OK', run.Raw(';'),
+                'fi',
+                ],
+            wait=False,
+            stdout=StringIO(),
+            )
+        run.wait(processes)
+        if any(r.stdout.getvalue() != 'OK' for r in processes):
+            log.warning('Found coredumps, flagging run as failed.')
+            ctx.summary['success'] = False

diff --git a/teuthology/task/kclient.py b/teuthology/task/kclient.py
index 56fe2a452eb580f55cc45f07627bc3fa9268ecea..806c2bfe6b1ba198b5f379407afdc415984365eb 100644 (file)
--- a/teuthology/task/kclient.py
+++ b/teuthology/task/kclient.py
@@ -71,6 +71,7 @@ def task(ctx, config):
         remote.run(
             args=[
                 'sudo',
+                '/tmp/cephtest/enable-coredump',
                 '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                 '/tmp/cephtest/archive/coverage',
                 '/tmp/cephtest/binary/usr/local/sbin/mount.ceph',