From: Tommi Virtanen Date: Fri, 17 Jun 2011 23:00:39 +0000 (-0700) Subject: Archive cores dumped during test, record test as failed if any seen. X-Git-Tag: v0.94.10~27^2^2~364^2~1705 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=57c542b9e8534a89080b61a3d4484596c6cfd1e8;p=ceph.git Archive cores dumped during test, record test as failed if any seen. --- diff --git a/teuthology/misc.py b/teuthology/misc.py index 5b8fc26472f9..25f77726ada8 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -169,6 +169,7 @@ def create_simple_monmap(remote, conf): log.debug('Ceph mon addresses: %s', addresses) args = [ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', '/tmp/cephtest/archive/coverage', '/tmp/cephtest/binary/usr/local/bin/monmaptool', @@ -216,6 +217,7 @@ def wait_until_healthy(remote): while True: r = remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', '/tmp/cephtest/archive/coverage', '/tmp/cephtest/binary/usr/local/bin/ceph', @@ -258,6 +260,7 @@ def wait_until_fuse_mounted(remote, fuse, mountpoint): def write_secret_file(remote, role, filename): remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', '/tmp/cephtest/archive/coverage', '/tmp/cephtest/binary/usr/local/bin/cauthtool', diff --git a/teuthology/run.py b/teuthology/run.py index 11b5fd7240ef..c27e4a9b053d 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -96,6 +96,7 @@ def main(): {'internal.check_conflict': None}, {'internal.base': None}, {'internal.archive': None}, + {'internal.coredump': None}, ] from teuthology.run_tasks import run_tasks diff --git a/teuthology/task/ceph.py b/teuthology/task/ceph.py index 8b7bef86aa3d..d3ffc7bdb813 100644 --- a/teuthology/task/ceph.py +++ b/teuthology/task/ceph.py @@ -58,7 +58,7 @@ def ceph_log(ctx, config): @contextlib.contextmanager def ship_utilities(ctx, config): assert config is None - FILES = ['daemon-helper'] + FILES = ['daemon-helper', 'enable-coredump'] for filename in FILES: log.info('Shipping %r...', filename) src = os.path.join(os.path.dirname(__file__), filename) @@ -185,6 +185,7 @@ def cluster(ctx, config): log.info('Setting up mon.0...') ctx.cluster.only('mon.0').run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', @@ -194,6 +195,7 @@ def cluster(ctx, config): ) ctx.cluster.only('mon.0').run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', @@ -211,6 +213,7 @@ def cluster(ctx, config): log.info('Creating admin key on mon.0...') ctx.cluster.only('mon.0').run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', @@ -254,6 +257,7 @@ def cluster(ctx, config): run.wait( mons.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/osdmaptool', @@ -275,6 +279,7 @@ def cluster(ctx, config): for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', @@ -291,6 +296,7 @@ def cluster(ctx, config): for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', @@ -307,6 +313,7 @@ def cluster(ctx, config): for id_ in teuthology.roles_of_type(roles_for_host, 'client'): remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', @@ -351,6 +358,7 @@ def cluster(ctx, config): run.wait( mons.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', @@ -369,6 +377,7 @@ def cluster(ctx, config): for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cmon', @@ -392,6 +401,7 @@ def cluster(ctx, config): ) remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cosd', @@ -448,6 +458,7 @@ def mon(ctx, config): for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): proc = remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/daemon-helper', @@ -489,6 +500,7 @@ def osd(ctx, config): for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): proc = remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/daemon-helper', @@ -530,6 +542,7 @@ def mds(ctx, config): for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): proc = remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/daemon-helper', diff --git a/teuthology/task/cfuse.py b/teuthology/task/cfuse.py index 23e263690339..988fca18da66 100644 --- a/teuthology/task/cfuse.py +++ b/teuthology/task/cfuse.py @@ -63,6 +63,7 @@ def task(ctx, config): ) proc = remote.run( args=[ + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', '/tmp/cephtest/archive/coverage', '/tmp/cephtest/daemon-helper', diff --git a/teuthology/task/enable-coredump b/teuthology/task/enable-coredump new file mode 100755 index 000000000000..42577648fcb4 --- /dev/null +++ b/teuthology/task/enable-coredump @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +ulimit -c unlimited +exec "$@" diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py index 937f1d4c1b69..7722ea63eba8 100644 --- a/teuthology/task/internal.py +++ b/teuthology/task/internal.py @@ -1,3 +1,4 @@ +from cStringIO import StringIO import contextlib import gevent import logging @@ -134,3 +135,52 @@ def archive(ctx, config): wait=False, ), ) + +@contextlib.contextmanager +def coredump(ctx, config): + log.info('Enabling coredump saving...') + run.wait( + ctx.cluster.run( + args=[ + 'install', '-d', '-m0755', '--', + '/tmp/cephtest/archive/coredump', + run.Raw('&&'), + 'sudo', 'sysctl', '-w', 'kernel.core_pattern=/tmp/cephtest/archive/coredump/%t.%p.core', + ], + wait=False, + ) + ) + + try: + yield + finally: + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'sysctl', '-w', 'kernel.core_pattern=core', + run.Raw('&&'), + # don't litter the archive dir if there were no cores dumped + 'rmdir', + '--ignore-fail-on-non-empty', + '--', + '/tmp/cephtest/archive/coredump', + ], + wait=False, + ) + ) + + # set success=false if the dir is still there = coredumps were + # seen + processes = ctx.cluster.run( + args=[ + 'if', 'test', '!', '-e', '/tmp/cephtest/archive/coredump', run.Raw(';'), 'then', + 'echo', 'OK', run.Raw(';'), + 'fi', + ], + wait=False, + stdout=StringIO(), + ) + run.wait(processes) + if any(r.stdout.getvalue() != 'OK' for r in processes): + log.warning('Found coredumps, flagging run as failed.') + ctx.summary['success'] = False diff --git a/teuthology/task/kclient.py b/teuthology/task/kclient.py index 56fe2a452eb5..806c2bfe6b1b 100644 --- a/teuthology/task/kclient.py +++ b/teuthology/task/kclient.py @@ -71,6 +71,7 @@ def task(ctx, config): remote.run( args=[ 'sudo', + '/tmp/cephtest/enable-coredump', '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', '/tmp/cephtest/archive/coverage', '/tmp/cephtest/binary/usr/local/sbin/mount.ceph',