From 14ddb419d6d49704458c021c1c153e4aa6229478 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Wed, 15 Jun 2011 14:57:02 -0700 Subject: [PATCH] Start splitting the ceph task into smaller units. This lets them nicely have independent error handling; an example case where that is desirable is that the old code failed to archive the logs if one of the daemons exited with an error. This also means a config file might specify just some of the smaller tasks, e.g. for a test that explicitly wants to create operational error situations. The diff looks big because of an indentation change, use "git diff -b" for a smaller diff. --- teuthology/contextutil.py | 36 ++ teuthology/task/ceph.py | 911 ++++++++++++++++++++------------------ 2 files changed, 517 insertions(+), 430 deletions(-) create mode 100644 teuthology/contextutil.py diff --git a/teuthology/contextutil.py b/teuthology/contextutil.py new file mode 100644 index 0000000000..296d889690 --- /dev/null +++ b/teuthology/contextutil.py @@ -0,0 +1,36 @@ +import contextlib +import sys + +@contextlib.contextmanager +def nested(*managers): + """ + Like contextlib.nested but takes callables returning context + managers, to avoid the major reason why contextlib.nested was + deprecated. + """ + exits = [] + vars = [] + exc = (None, None, None) + try: + for mgr_fn in managers: + mgr = mgr_fn() + exit = mgr.__exit__ + enter = mgr.__enter__ + vars.append(enter()) + exits.append(exit) + yield vars + except: + exc = sys.exc_info() + finally: + while exits: + exit = exits.pop() + try: + if exit(*exc): + exc = (None, None, None) + except: + exc = sys.exc_info() + if exc != (None, None, None): + # Don't rely on sys.exc_info() still containing + # the right information. Another exception may + # have been raised and caught by an exit method + raise exc[0], exc[1], exc[2] diff --git a/teuthology/task/ceph.py b/teuthology/task/ceph.py index f0a388fb5c..37620995ee 100644 --- a/teuthology/task/ceph.py +++ b/teuthology/task/ceph.py @@ -8,10 +8,103 @@ import tarfile from teuthology import misc as teuthology from teuthology import safepath +from teuthology import contextutil from orchestra import run log = logging.getLogger(__name__) + +@contextlib.contextmanager +def ship_utilities(ctx, config): + assert config is None + FILES = ['daemon-helper'] + for filename in FILES: + log.info('Shipping %r...', filename) + src = os.path.join(os.path.dirname(__file__), filename) + dst = os.path.join('/tmp/cephtest', filename) + with file(src, 'rb') as f: + for rem in ctx.cluster.remotes.iterkeys(): + teuthology.write_file( + remote=rem, + path=dst, + data=f, + ) + f.seek(0) + rem.run( + args=[ + 'chmod', + 'a=rx', + '--', + dst, + ], + ) + + try: + yield + finally: + log.info('Removing shipped files: %s...', ' '.join(FILES)) + filenames = ( + os.path.join('/tmp/cephtest', filename) + for filename in FILES + ) + run.wait( + ctx.cluster.run( + args=[ + 'rm', + '-rf', + '--', + ] + list(filenames), + wait=False, + ), + ) + +@contextlib.contextmanager +def binaries(ctx, config): + log.info('Unpacking ceph binaries...') + sha1, ceph_bindir_url = teuthology.get_ceph_binary_url( + branch=config.get('branch'), + tag=config.get('tag'), + sha1=config.get('sha1'), + flavor=config.get('flavor'), + ) + if ctx.archive is not None: + with file(os.path.join(ctx.archive, 'ceph-sha1'), 'w') as f: + f.write(sha1 + '\n') + ctx.cluster.run( + args=[ + 'install', '-d', '-m0755', '--', '/tmp/cephtest/binary', + run.Raw('&&'), + 'uname', '-m', + run.Raw('|'), + 'sed', '-e', 's/^/ceph./; s/$/.tgz/', + run.Raw('|'), + 'wget', + '-nv', + '-O-', + '--base={url}'.format(url=ceph_bindir_url), + # need to use --input-file to make wget respect --base + '--input-file=-', + run.Raw('|'), + 'tar', '-xzf', '-', '-C', '/tmp/cephtest/binary', + ], + ) + try: + yield + finally: + log.info('Removing ceph binaries...') + run.wait( + ctx.cluster.run( + args=[ + 'rm', + '-rf', + '--', + '/tmp/cephtest/binary', + ], + wait=False, + ), + ) + + @contextlib.contextmanager def task(ctx, config): """ @@ -84,7 +177,6 @@ def task(ctx, config): ctx.cluster.run( args=[ 'install', '-d', '-m0755', '--', - '/tmp/cephtest/binary', '/tmp/cephtest/archive', '/tmp/cephtest/archive/log', '/tmp/cephtest/archive/profiling-logger', @@ -95,290 +187,273 @@ def task(ctx, config): ) ) - for filename in ['daemon-helper']: - log.info('Shipping %r...', filename) - src = os.path.join(os.path.dirname(__file__), filename) - dst = os.path.join('/tmp/cephtest', filename) - with file(src, 'rb') as f: - for rem in ctx.cluster.remotes.iterkeys(): - teuthology.write_file( - remote=rem, - path=dst, - data=f, - ) - f.seek(0) - rem.run( - args=[ - 'chmod', - 'a=rx', - '--', - dst, - ], - ) - - log.info('Untarring ceph binaries...') - sha1, ceph_bindir_url = teuthology.get_ceph_binary_url( - branch=config.get('branch'), - tag=config.get('tag'), - sha1=config.get('sha1'), - flavor=flavor, - ) - ctx.cluster.run( - args=[ - 'uname', '-m', - run.Raw('|'), - 'sed', '-e', 's/^/ceph./; s/$/.tgz/', - run.Raw('|'), - 'wget', - '-nv', - '-O-', - '--base={url}'.format(url=ceph_bindir_url), - # need to use --input-file to make wget respect --base - '--input-file=-', - run.Raw('|'), - 'tar', '-xzf', '-', '-C', '/tmp/cephtest/binary', - ], - ) - - log.info('Writing configs...') - remotes_and_roles = ctx.cluster.remotes.items() - roles = [roles for (remote, roles) in remotes_and_roles] - ips = [host for (host, port) in (remote.ssh.get_transport().getpeername() for (remote, roles) in remotes_and_roles)] - conf = teuthology.skeleton_config(roles=roles, ips=ips) - conf_fp = StringIO() - conf.write(conf_fp) - conf_fp.seek(0) - writes = ctx.cluster.run( - args=[ - 'python', - '-c', - 'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))', - '/tmp/cephtest/ceph.conf', - ], - stdin=run.PIPE, - wait=False, - ) - teuthology.feed_many_stdins_and_close(conf_fp, writes) - run.wait(writes) - - log.info('Setting up mon.0...') - ctx.cluster.only('mon.0').run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '--create-keyring', - '/tmp/cephtest/ceph.keyring', - ], - ) - ctx.cluster.only('mon.0').run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '--gen-key', - '--name=mon.', - '/tmp/cephtest/ceph.keyring', - ], - ) - (mon0_remote,) = ctx.cluster.only('mon.0').remotes.keys() - teuthology.create_simple_monmap( - remote=mon0_remote, - conf=conf, - ) - - log.info('Creating admin key on mon.0...') - ctx.cluster.only('mon.0').run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '--gen-key', - '--name=client.admin', - '--set-uid=0', - '--cap', 'mon', 'allow *', - '--cap', 'osd', 'allow *', - '--cap', 'mds', 'allow', - '/tmp/cephtest/ceph.keyring', - ], - ) + with contextutil.nested( + lambda: ship_utilities(ctx=ctx, config=None), + lambda: binaries(ctx=ctx, config=dict( + branch=config.get('branch'), + tag=config.get('tag'), + sha1=config.get('sha1'), + flavor=flavor, + )), + ): + log.info('Writing configs...') + remotes_and_roles = ctx.cluster.remotes.items() + roles = [roles for (remote, roles) in remotes_and_roles] + ips = [host for (host, port) in (remote.ssh.get_transport().getpeername() for (remote, roles) in remotes_and_roles)] + conf = teuthology.skeleton_config(roles=roles, ips=ips) + conf_fp = StringIO() + conf.write(conf_fp) + conf_fp.seek(0) + writes = ctx.cluster.run( + args=[ + 'python', + '-c', + 'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))', + '/tmp/cephtest/ceph.conf', + ], + stdin=run.PIPE, + wait=False, + ) + teuthology.feed_many_stdins_and_close(conf_fp, writes) + run.wait(writes) - log.info('Copying mon.0 info to all monitors...') - keyring = teuthology.get_file( - remote=mon0_remote, - path='/tmp/cephtest/ceph.keyring', - ) - monmap = teuthology.get_file( - remote=mon0_remote, - path='/tmp/cephtest/monmap', - ) - mons = ctx.cluster.only(teuthology.is_type('mon')) - mons_no_0 = mons.exclude('mon.0') - - for rem in mons_no_0.remotes.iterkeys(): - # copy mon key and initial monmap - log.info('Sending mon0 info to node {remote}'.format(remote=rem)) - teuthology.write_file( - remote=rem, - path='/tmp/cephtest/ceph.keyring', - data=keyring, + log.info('Setting up mon.0...') + ctx.cluster.only('mon.0').run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/binary/usr/local/bin/cauthtool', + '--create-keyring', + '/tmp/cephtest/ceph.keyring', + ], ) - teuthology.write_file( - remote=rem, - path='/tmp/cephtest/monmap', - data=monmap, + ctx.cluster.only('mon.0').run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/binary/usr/local/bin/cauthtool', + '--gen-key', + '--name=mon.', + '/tmp/cephtest/ceph.keyring', + ], + ) + (mon0_remote,) = ctx.cluster.only('mon.0').remotes.keys() + teuthology.create_simple_monmap( + remote=mon0_remote, + conf=conf, ) - log.info('Setting up mon nodes...') - run.wait( - mons.run( + log.info('Creating admin key on mon.0...') + ctx.cluster.only('mon.0').run( args=[ '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/osdmaptool', - '--clobber', - '--createsimple', '{num:d}'.format( - num=teuthology.num_instances_of_type(ctx.cluster, 'osd'), - ), - '/tmp/cephtest/osdmap', - '--pg_bits', '2', - '--pgp_bits', '4', + '/tmp/cephtest/binary/usr/local/bin/cauthtool', + '--gen-key', + '--name=client.admin', + '--set-uid=0', + '--cap', 'mon', 'allow *', + '--cap', 'osd', 'allow *', + '--cap', 'mds', 'allow', + '/tmp/cephtest/ceph.keyring', ], - wait=False, - ), - ) + ) - for remote, roles_for_host in mons.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): - remote.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cmon', - '--mkfs', - '-i', id_, - '-c', '/tmp/cephtest/ceph.conf', - '--monmap=/tmp/cephtest/monmap', - '--osdmap=/tmp/cephtest/osdmap', - '--keyring=/tmp/cephtest/ceph.keyring', - ], + log.info('Copying mon.0 info to all monitors...') + keyring = teuthology.get_file( + remote=mon0_remote, + path='/tmp/cephtest/ceph.keyring', + ) + monmap = teuthology.get_file( + remote=mon0_remote, + path='/tmp/cephtest/monmap', + ) + mons = ctx.cluster.only(teuthology.is_type('mon')) + mons_no_0 = mons.exclude('mon.0') + + for rem in mons_no_0.remotes.iterkeys(): + # copy mon key and initial monmap + log.info('Sending mon0 info to node {remote}'.format(remote=rem)) + teuthology.write_file( + remote=rem, + path='/tmp/cephtest/ceph.keyring', + data=keyring, + ) + teuthology.write_file( + remote=rem, + path='/tmp/cephtest/monmap', + data=monmap, ) - run.wait( - mons.run( - args=[ - 'rm', - '--', - '/tmp/cephtest/monmap', - '/tmp/cephtest/osdmap', - ], - wait=False, - ), - ) - - mon_daemons = {} - log.info('Starting mon daemons...') - for remote, roles_for_host in mons.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): - proc = remote.run( + log.info('Setting up mon nodes...') + run.wait( + mons.run( args=[ '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, - '/tmp/cephtest/daemon-helper', - daemon_signal, - '/tmp/cephtest/binary/usr/local/bin/cmon', - '-f', - '-i', id_, - '-c', '/tmp/cephtest/ceph.conf', + '/tmp/cephtest/binary/usr/local/bin/osdmaptool', + '--clobber', + '--createsimple', '{num:d}'.format( + num=teuthology.num_instances_of_type(ctx.cluster, 'osd'), + ), + '/tmp/cephtest/osdmap', + '--pg_bits', '2', + '--pgp_bits', '4', ], - logger=log.getChild('mon.{id}'.format(id=id_)), - stdin=run.PIPE, wait=False, - ) - mon_daemons[id_] = proc + ), + ) - log.info('Setting up osd nodes...') - osds = ctx.cluster.only(teuthology.is_type('osd')) - for remote, roles_for_host in osds.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): - remote.run( + for remote, roles_for_host in mons.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/binary/usr/local/bin/cmon', + '--mkfs', + '-i', id_, + '-c', '/tmp/cephtest/ceph.conf', + '--monmap=/tmp/cephtest/monmap', + '--osdmap=/tmp/cephtest/osdmap', + '--keyring=/tmp/cephtest/ceph.keyring', + ], + ) + + run.wait( + mons.run( args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '--create-keyring', - '--gen-key', - '--name=osd.{id}'.format(id=id_), - '/tmp/cephtest/data/osd.{id}.keyring'.format(id=id_), + 'rm', + '--', + '/tmp/cephtest/monmap', + '/tmp/cephtest/osdmap', ], - ) + wait=False, + ), + ) + + mon_daemons = {} + log.info('Starting mon daemons...') + for remote, roles_for_host in mons.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): + proc = remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/daemon-helper', + daemon_signal, + '/tmp/cephtest/binary/usr/local/bin/cmon', + '-f', + '-i', id_, + '-c', '/tmp/cephtest/ceph.conf', + ], + logger=log.getChild('mon.{id}'.format(id=id_)), + stdin=run.PIPE, + wait=False, + ) + mon_daemons[id_] = proc - log.info('Setting up mds nodes...') - mdss = ctx.cluster.only(teuthology.is_type('mds')) - for remote, roles_for_host in mdss.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): - remote.run( + log.info('Setting up osd nodes...') + osds = ctx.cluster.only(teuthology.is_type('osd')) + for remote, roles_for_host in osds.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/binary/usr/local/bin/cauthtool', + '--create-keyring', + '--gen-key', + '--name=osd.{id}'.format(id=id_), + '/tmp/cephtest/data/osd.{id}.keyring'.format(id=id_), + ], + ) + + log.info('Setting up mds nodes...') + mdss = ctx.cluster.only(teuthology.is_type('mds')) + for remote, roles_for_host in mdss.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/binary/usr/local/bin/cauthtool', + '--create-keyring', + '--gen-key', + '--name=mds.{id}'.format(id=id_), + '/tmp/cephtest/data/mds.{id}.keyring'.format(id=id_), + ], + ) + + log.info('Setting up client nodes...') + clients = ctx.cluster.only(teuthology.is_type('client')) + for remote, roles_for_host in clients.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'client'): + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/binary/usr/local/bin/cauthtool', + '--create-keyring', + '--gen-key', + # TODO this --name= is not really obeyed, all unknown "types" are munged to "client" + '--name=client.{id}'.format(id=id_), + '/tmp/cephtest/data/client.{id}.keyring'.format(id=id_), + ], + ) + + log.info('Reading keys from all nodes...') + keys = [] + for remote, roles_for_host in ctx.cluster.remotes.iteritems(): + for type_ in ['osd','mds','client']: + for id_ in teuthology.roles_of_type(roles_for_host, type_): + data = teuthology.get_file( + remote=remote, + path='/tmp/cephtest/data/{type}.{id}.keyring'.format( + type=type_, + id=id_, + ), + ) + keys.append((type_, id_, data)) + + log.info('Adding keys to mon.0...') + for type_, id_, data in keys: + teuthology.write_file( + remote=mon0_remote, + path='/tmp/cephtest/temp.keyring', + data=data, + ) + mon0_remote.run( args=[ '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '--create-keyring', - '--gen-key', - '--name=mds.{id}'.format(id=id_), - '/tmp/cephtest/data/mds.{id}.keyring'.format(id=id_), - ], + '/tmp/cephtest/temp.keyring', + '--name={type}.{id}'.format( + type=type_, + id=id_, + ), + ] + list(teuthology.generate_caps(type_)), ) - - log.info('Setting up client nodes...') - clients = ctx.cluster.only(teuthology.is_type('client')) - for remote, roles_for_host in clients.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'client'): - remote.run( + mon0_remote.run( args=[ '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '--create-keyring', - '--gen-key', - # TODO this --name= is not really obeyed, all unknown "types" are munged to "client" - '--name=client.{id}'.format(id=id_), - '/tmp/cephtest/data/client.{id}.keyring'.format(id=id_), - ], - ) - - log.info('Reading keys from all nodes...') - keys = [] - for remote, roles_for_host in ctx.cluster.remotes.iteritems(): - for type_ in ['osd','mds','client']: - for id_ in teuthology.roles_of_type(roles_for_host, type_): - data = teuthology.get_file( - remote=remote, - path='/tmp/cephtest/data/{type}.{id}.keyring'.format( + '/tmp/cephtest/binary/usr/local/bin/ceph', + '-c', '/tmp/cephtest/ceph.conf', + '-k', '/tmp/cephtest/ceph.keyring', + '-i', '/tmp/cephtest/temp.keyring', + 'auth', + 'add', + '{type}.{id}'.format( type=type_, id=id_, ), - ) - keys.append((type_, id_, data)) + ], + ) - log.info('Adding keys to mon.0...') - for type_, id_, data in keys: - teuthology.write_file( - remote=mon0_remote, - path='/tmp/cephtest/temp.keyring', - data=data, - ) - mon0_remote.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '/tmp/cephtest/temp.keyring', - '--name={type}.{id}'.format( - type=type_, - id=id_, - ), - ] + list(teuthology.generate_caps(type_)), - ) + log.info('Setting max_mds...') + # TODO where does this belong? mon0_remote.run( args=[ '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', @@ -386,208 +461,184 @@ def task(ctx, config): '/tmp/cephtest/binary/usr/local/bin/ceph', '-c', '/tmp/cephtest/ceph.conf', '-k', '/tmp/cephtest/ceph.keyring', - '-i', '/tmp/cephtest/temp.keyring', - 'auth', - 'add', - '{type}.{id}'.format( - type=type_, - id=id_, + 'mds', + 'set_max_mds', + '{num_mds:d}'.format( + num_mds=teuthology.num_instances_of_type(ctx.cluster, 'mds'), ), ], ) - log.info('Setting max_mds...') - # TODO where does this belong? - mon0_remote.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/ceph', - '-c', '/tmp/cephtest/ceph.conf', - '-k', '/tmp/cephtest/ceph.keyring', - 'mds', - 'set_max_mds', - '{num_mds:d}'.format( - num_mds=teuthology.num_instances_of_type(ctx.cluster, 'mds'), - ), - ], - ) + log.info('Running mkfs on osd nodes...') + for remote, roles_for_host in osds.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): + remote.run( + args=[ + 'mkdir', + os.path.join('/tmp/cephtest/data', 'osd.{id}.data'.format(id=id_)), + ], + ) + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/binary/usr/local/bin/cosd', + '--mkfs', + '-i', id_, + '-c', '/tmp/cephtest/ceph.conf', + ], + ) - log.info('Running mkfs on osd nodes...') - for remote, roles_for_host in osds.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): - remote.run( - args=[ - 'mkdir', - os.path.join('/tmp/cephtest/data', 'osd.{id}.data'.format(id=id_)), - ], - ) - remote.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/binary/usr/local/bin/cosd', - '--mkfs', - '-i', id_, - '-c', '/tmp/cephtest/ceph.conf', - ], - ) + osd_daemons = {} + log.info('Starting osd daemons...') + for remote, roles_for_host in osds.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): + proc = remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/daemon-helper', + daemon_signal, + '/tmp/cephtest/binary/usr/local/bin/cosd', + '-f', + '-i', id_, + '-c', '/tmp/cephtest/ceph.conf', + ], + logger=log.getChild('osd.{id}'.format(id=id_)), + stdin=run.PIPE, + wait=False, + ) + osd_daemons[id_] = proc - osd_daemons = {} - log.info('Starting osd daemons...') - for remote, roles_for_host in osds.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): - proc = remote.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/daemon-helper', - daemon_signal, - '/tmp/cephtest/binary/usr/local/bin/cosd', - '-f', - '-i', id_, - '-c', '/tmp/cephtest/ceph.conf', - ], - logger=log.getChild('osd.{id}'.format(id=id_)), - stdin=run.PIPE, - wait=False, - ) - osd_daemons[id_] = proc + mds_daemons = {} + log.info('Starting mds daemons...') + for remote, roles_for_host in mdss.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): + proc = remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + coverage_dir, + '/tmp/cephtest/daemon-helper', + daemon_signal, + '/tmp/cephtest/binary/usr/local/bin/cmds', + '-f', + '-i', id_, + '-c', '/tmp/cephtest/ceph.conf', + ], + logger=log.getChild('mds.{id}'.format(id=id_)), + stdin=run.PIPE, + wait=False, + ) + mds_daemons[id_] = proc - mds_daemons = {} - log.info('Starting mds daemons...') - for remote, roles_for_host in mdss.remotes.iteritems(): - for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): - proc = remote.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', - coverage_dir, - '/tmp/cephtest/daemon-helper', - daemon_signal, - '/tmp/cephtest/binary/usr/local/bin/cmds', - '-f', - '-i', id_, - '-c', '/tmp/cephtest/ceph.conf', - ], - logger=log.getChild('mds.{id}'.format(id=id_)), - stdin=run.PIPE, - wait=False, - ) - mds_daemons[id_] = proc + log.info('Waiting until ceph is healthy...') + teuthology.wait_until_healthy( + remote=mon0_remote, + ) - log.info('Waiting until ceph is healthy...') - teuthology.wait_until_healthy( - remote=mon0_remote, - ) + try: + yield + finally: + log.info('Shutting down mds daemons...') + for id_, proc in mds_daemons.iteritems(): + proc.stdin.close() - try: - yield - finally: - log.info('Shutting down mds daemons...') - for id_, proc in mds_daemons.iteritems(): - proc.stdin.close() + log.info('Shutting down osd daemons...') + for id_, proc in osd_daemons.iteritems(): + proc.stdin.close() - log.info('Shutting down osd daemons...') - for id_, proc in osd_daemons.iteritems(): - proc.stdin.close() + log.info('Shutting down mon daemons...') + for id_, proc in mon_daemons.iteritems(): + proc.stdin.close() - log.info('Shutting down mon daemons...') - for id_, proc in mon_daemons.iteritems(): - proc.stdin.close() + run.wait(mds_daemons.itervalues()) + run.wait(osd_daemons.itervalues()) + run.wait(mon_daemons.itervalues()) - run.wait(mds_daemons.itervalues()) - run.wait(osd_daemons.itervalues()) - run.wait(mon_daemons.itervalues()) + log.info('Removing uninteresting files...') + run.wait( + ctx.cluster.run( + args=[ + 'rm', + '-rf', + '--', + '/tmp/cephtest/ceph.conf', + '/tmp/cephtest/ceph.keyring', + '/tmp/cephtest/temp.keyring', + '/tmp/cephtest/data', + ], + wait=False, + ), + ) - log.info('Removing uninteresting files...') + if ctx.archive is not None: + log.info('Compressing logs...') run.wait( ctx.cluster.run( args=[ - 'rm', - '-rf', + 'find', + '/tmp/cephtest/archive/log', + '-name', + '*.log', + '-print0', + run.Raw('|'), + 'xargs', + '-0', + '--no-run-if-empty', + '--', + 'bzip2', + '-9', '--', - '/tmp/cephtest/binary', - '/tmp/cephtest/daemon-helper', - '/tmp/cephtest/ceph.conf', - '/tmp/cephtest/ceph.keyring', - '/tmp/cephtest/temp.keyring', - '/tmp/cephtest/data', ], wait=False, ), ) - if ctx.archive is not None: - with file(os.path.join(ctx.archive, 'ceph-sha1'), 'w') as f: - f.write(sha1 + '\n') - - log.info('Compressing logs...') - run.wait( - ctx.cluster.run( - args=[ - 'find', - '/tmp/cephtest/archive/log', - '-name', - '*.log', - '-print0', - run.Raw('|'), - 'xargs', - '-0', - '--no-run-if-empty', - '--', - 'bzip2', - '-9', - '--', - ], - wait=False, - ), + log.info('Transferring archived files...') + logdir = os.path.join(ctx.archive, 'remote') + os.mkdir(logdir) + for remote in ctx.cluster.remotes.iterkeys(): + path = os.path.join(logdir, remote.shortname) + os.mkdir(path) + log.debug('Transferring archived files from %s to %s', remote.shortname, path) + proc = remote.run( + args=[ + 'tar', + 'c', + '-f', '-', + '-C', '/tmp/cephtest/archive', + '--', + '.', + ], + stdout=run.PIPE, + wait=False, ) - - log.info('Transferring archived files...') - logdir = os.path.join(ctx.archive, 'remote') - os.mkdir(logdir) - for remote in ctx.cluster.remotes.iterkeys(): - path = os.path.join(logdir, remote.shortname) - os.mkdir(path) - log.debug('Transferring archived files from %s to %s', remote.shortname, path) - proc = remote.run( - args=[ - 'tar', - 'c', - '-f', '-', - '-C', '/tmp/cephtest/archive', - '--', - '.', - ], - stdout=run.PIPE, - wait=False, - ) - tar = tarfile.open(mode='r|', fileobj=proc.stdout) - while True: - ti = tar.next() - if ti is None: - break - - if ti.isdir(): - # ignore silently; easier to just create leading dirs below - pass - elif ti.isfile(): - sub = safepath.munge(ti.name) - safepath.makedirs(root=path, path=os.path.dirname(sub)) - tar.makefile(ti, targetpath=os.path.join(path, sub)) + tar = tarfile.open(mode='r|', fileobj=proc.stdout) + while True: + ti = tar.next() + if ti is None: + break + + if ti.isdir(): + # ignore silently; easier to just create leading dirs below + pass + elif ti.isfile(): + sub = safepath.munge(ti.name) + safepath.makedirs(root=path, path=os.path.dirname(sub)) + tar.makefile(ti, targetpath=os.path.join(path, sub)) + else: + if ti.isdev(): + type_ = 'device' + elif ti.issym(): + type_ = 'symlink' + elif ti.islnk(): + type_ = 'hard link' else: - if ti.isdev(): - type_ = 'device' - elif ti.issym(): - type_ = 'symlink' - elif ti.islnk(): - type_ = 'hard link' - else: - type_ = 'unknown' - log.info('Ignoring tar entry: %r type %r', ti.name, type_) - continue - proc.exitstatus.get() + type_ = 'unknown' + log.info('Ignoring tar entry: %r type %r', ti.name, type_) + continue + proc.exitstatus.get() log.info('Removing archived files...') run.wait( -- 2.39.5