From 57f979f1773083480a634f79b284057a5917f338 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Fri, 3 Jun 2011 14:47:44 -0700 Subject: [PATCH] Refactor for modularity. New style: run "./virtualenv/bin/teuthology -v interactive.yaml". --- dbench.yaml | 1 + interactive.yaml | 12 + requirements.txt | 1 + setup.py | 7 + teuthology/misc.py | 14 +- teuthology/run.py | 71 ++++++ dbench.py => teuthology/task/ceph.py | 235 ++++++++---------- .../task/daemon-helper | 0 8 files changed, 201 insertions(+), 140 deletions(-) create mode 100644 interactive.yaml create mode 100644 teuthology/run.py rename dbench.py => teuthology/task/ceph.py (67%) rename daemon-helper => teuthology/task/daemon-helper (100%) diff --git a/dbench.yaml b/dbench.yaml index c69626e6ce516..a8d5b0f964a22 100644 --- a/dbench.yaml +++ b/dbench.yaml @@ -7,6 +7,7 @@ targets: - ubuntu@sepia71.ceph.dreamhost.com - ubuntu@sepia72.ceph.dreamhost.com tasks: +- ceph: - cfuse: [client.0] - autotest: client.0: [dbench] diff --git a/interactive.yaml b/interactive.yaml new file mode 100644 index 0000000000000..df5ad0d891b48 --- /dev/null +++ b/interactive.yaml @@ -0,0 +1,12 @@ +roles: +- [mon.0, mds.0, osd.0] +- [mon.1, osd.1] +- [mon.2, client.0] +targets: +- ubuntu@sepia70.ceph.dreamhost.com +- ubuntu@sepia71.ceph.dreamhost.com +- ubuntu@sepia72.ceph.dreamhost.com +tasks: +- ceph: +- cfuse: [client.0] +- interactive: diff --git a/requirements.txt b/requirements.txt index c44846a98ea10..83e9769c61be4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ configobj PyYAML bunch >=1.0.0 +argparse >=1.2.1 diff --git a/setup.py b/setup.py index a2d408c8f0ce8..8421bdeac7727 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,13 @@ setup( 'configobj', 'PyYAML', 'bunch >=1.0.0', + 'argparse >=1.2.1' ], + entry_points={ + 'console_scripts': [ + 'teuthology = teuthology.run:main', + ], + }, + ) diff --git a/teuthology/misc.py b/teuthology/misc.py index ca02099d054b2..c230aef894fbd 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -93,11 +93,6 @@ def roles_of_type(roles_for_host, type_): id_ = name[len(prefix):] yield id_ -def all_roles_of_type(roles, type_): - for roles_for_host in roles: - for id_ in roles_of_type(roles_for_host, type_): - yield id_ - def is_type(type_): """ Returns a matcher function for whether role is of type given. @@ -107,16 +102,13 @@ def is_type(type_): return role.startswith(prefix) return _is_type -def num_instances_of_type(roles, type_): +def num_instances_of_type(cluster, type_): + remotes_and_roles = cluster.remotes.items() + roles = [roles for (remote, roles) in remotes_and_roles] prefix = '{type}.'.format(type=type_) num = sum(sum(1 for role in hostroles if role.startswith(prefix)) for hostroles in roles) return num -def server_with_role(all_roles, role): - for idx, host_roles in enumerate(all_roles): - if role in host_roles: - return idx - def create_simple_monmap(remote, conf): """ Writes a simple monmap based on current ceph.conf into /monmap. diff --git a/teuthology/run.py b/teuthology/run.py new file mode 100644 index 0000000000000..3a0fe29bf0f3a --- /dev/null +++ b/teuthology/run.py @@ -0,0 +1,71 @@ +import argparse +import yaml + +def config_file(string): + config = {} + try: + with file(string) as f: + g = yaml.safe_load_all(f) + for new in g: + config.update(new) + except IOError, e: + raise argparse.ArgumentTypeError(str(e)) + return config + +class MergeConfig(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + config = getattr(namespace, self.dest) + for new in values: + config.update(new) + +def parse_args(): + parser = argparse.ArgumentParser(description='Run ceph integration tests') + parser.add_argument( + '-v', '--verbose', + action='store_true', default=None, + help='be more verbose', + ) + parser.add_argument( + 'config', + metavar='CONFFILE', + nargs='+', + type=config_file, + action=MergeConfig, + default={}, + help='config file to read', + ) + + args = parser.parse_args() + return args + +def main(): + from gevent import monkey; monkey.patch_all() + from orchestra import monkey; monkey.patch_all() + + import logging + + log = logging.getLogger(__name__) + ctx = parse_args() + + loglevel = logging.INFO + if ctx.verbose: + loglevel = logging.DEBUG + + logging.basicConfig( + level=loglevel, + ) + + log.debug('\n '.join(['Config:', ] + yaml.safe_dump(ctx.config).splitlines())) + log.info('Opening connections...') + + from orchestra import connection, remote + import orchestra.cluster + + remotes = [remote.Remote(name=t, ssh=connection.connect(t)) + for t in ctx.config['targets']] + ctx.cluster = orchestra.cluster.Cluster() + for rem, roles in zip(remotes, ctx.config['roles']): + ctx.cluster.add(rem, roles) + + from teuthology.run_tasks import run_tasks + run_tasks(tasks=ctx.config['tasks'], ctx=ctx) diff --git a/dbench.py b/teuthology/task/ceph.py similarity index 67% rename from dbench.py rename to teuthology/task/ceph.py index 85903302f1576..35855187249f2 100644 --- a/dbench.py +++ b/teuthology/task/ceph.py @@ -1,59 +1,40 @@ -from gevent import monkey; monkey.patch_all() -from orchestra import monkey; monkey.patch_all() - from cStringIO import StringIO -import bunch +import contextlib import logging import os -import sys -import yaml +import gevent -from orchestra import connection, run, remote -import orchestra.cluster -# TODO cleanup -import teuthology.misc as teuthology -from teuthology.run_tasks import run_tasks +from teuthology import misc as teuthology +from orchestra import run log = logging.getLogger(__name__) -if __name__ == '__main__': - logging.basicConfig( - # level=logging.INFO, - level=logging.DEBUG, - ) - - with file('dbench.yaml') as f: - config = yaml.safe_load(f) - - ROLES = config['roles'] - - connections = [connection.connect(t) for t in config['targets']] - remotes = [remote.Remote(name=t, ssh=c) for c,t in zip(connections, config['targets'])] - cluster = orchestra.cluster.Cluster() - for rem, roles in zip(remotes, ROLES): - cluster.add(rem, roles) - - ctx = bunch.Bunch( - cluster=cluster, - ) +@contextlib.contextmanager +def task(ctx, config): + assert config is None log.info('Checking for old test directory...') - processes = cluster.run( + processes = ctx.cluster.run( args=[ 'test', '!', '-e', '/tmp/cephtest', ], wait=False, ) - try: - run.wait(processes) - except run.CommandFailedError as e: - log.error('Host %s has stale cephtest directory, check your lock and reboot to clean up.', rem) - sys.exit(1) + failed = False + for proc in processes: + assert isinstance(proc.exitstatus, gevent.event.AsyncResult) + try: + proc.exitstatus.get() + except run.CommandFailedError: + log.error('Host %s has stale cephtest directory, check your lock and reboot to clean up.', proc.remote.shortname) + failed = True + if failed: + raise RuntimeError('Stale jobs detected, aborting.') log.info('Creating directories...') run.wait( - cluster.run( + ctx.cluster.run( args=[ 'install', '-d', '-m0755', '--', '/tmp/cephtest/binary', @@ -71,7 +52,7 @@ if __name__ == '__main__': src = os.path.join(os.path.dirname(__file__), filename) dst = os.path.join('/tmp/cephtest', filename) with file(src, 'rb') as f: - for rem in cluster.remotes.iterkeys(): + for rem in ctx.cluster.remotes.iterkeys(): teuthology.write_file( remote=rem, path=dst, @@ -89,7 +70,7 @@ if __name__ == '__main__': log.info('Untarring ceph binaries...') ceph_bindir_url = teuthology.get_ceph_binary_url() - cluster.run( + ctx.cluster.run( args=[ 'uname', '-m', run.Raw('|'), @@ -107,12 +88,14 @@ if __name__ == '__main__': ) log.info('Writing configs...') - ips = [host for (host, port) in (conn.get_transport().getpeername() for conn in connections)] - conf = teuthology.skeleton_config(roles=ROLES, ips=ips) + remotes_and_roles = ctx.cluster.remotes.items() + roles = [roles for (remote, roles) in remotes_and_roles] + ips = [host for (host, port) in (remote.ssh.get_transport().getpeername() for (remote, roles) in remotes_and_roles)] + conf = teuthology.skeleton_config(roles=roles, ips=ips) conf_fp = StringIO() conf.write(conf_fp) conf_fp.seek(0) - writes = cluster.run( + writes = ctx.cluster.run( args=[ 'python', '-c', @@ -126,14 +109,14 @@ if __name__ == '__main__': run.wait(writes) log.info('Setting up mon.0...') - cluster.only('mon.0').run( + ctx.cluster.only('mon.0').run( args=[ '/tmp/cephtest/binary/usr/local/bin/cauthtool', '--create-keyring', '/tmp/cephtest/ceph.keyring', ], ) - cluster.only('mon.0').run( + ctx.cluster.only('mon.0').run( args=[ '/tmp/cephtest/binary/usr/local/bin/cauthtool', '--gen-key', @@ -141,14 +124,14 @@ if __name__ == '__main__': '/tmp/cephtest/ceph.keyring', ], ) - (mon0_remote,) = cluster.only('mon.0').remotes.keys() + (mon0_remote,) = ctx.cluster.only('mon.0').remotes.keys() teuthology.create_simple_monmap( remote=mon0_remote, conf=conf, ) log.info('Creating admin key on mon.0...') - cluster.only('mon.0').run( + ctx.cluster.only('mon.0').run( args=[ '/tmp/cephtest/binary/usr/local/bin/cauthtool', '--gen-key', @@ -170,7 +153,7 @@ if __name__ == '__main__': remote=mon0_remote, path='/tmp/cephtest/monmap', ) - mons = cluster.only(teuthology.is_type('mon')) + mons = ctx.cluster.only(teuthology.is_type('mon')) mons_no_0 = mons.exclude('mon.0') for rem in mons_no_0.remotes.iterkeys(): @@ -194,7 +177,7 @@ if __name__ == '__main__': '/tmp/cephtest/binary/usr/local/bin/osdmaptool', '--clobber', '--createsimple', '{num:d}'.format( - num=teuthology.num_instances_of_type(ROLES, 'osd'), + num=teuthology.num_instances_of_type(ctx.cluster, 'osd'), ), '/tmp/cephtest/osdmap', '--pg_bits', '2', @@ -204,19 +187,19 @@ if __name__ == '__main__': ), ) - for id_ in teuthology.all_roles_of_type(ROLES, 'mon'): - (rem,) = cluster.only('mon.{id}'.format(id=id_)).remotes.keys() - rem.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/cmon', - '--mkfs', - '-i', id_, - '-c', '/tmp/cephtest/ceph.conf', - '--monmap=/tmp/cephtest/monmap', - '--osdmap=/tmp/cephtest/osdmap', - '--keyring=/tmp/cephtest/ceph.keyring', - ], - ) + for remote, roles_for_host in mons.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/cmon', + '--mkfs', + '-i', id_, + '-c', '/tmp/cephtest/ceph.conf', + '--monmap=/tmp/cephtest/monmap', + '--osdmap=/tmp/cephtest/osdmap', + '--keyring=/tmp/cephtest/ceph.keyring', + ], + ) run.wait( mons.run( @@ -232,10 +215,9 @@ if __name__ == '__main__': mon_daemons = {} log.info('Starting mon daemons...') - for idx, roles_for_host in enumerate(ROLES): + for remote, roles_for_host in mons.remotes.iteritems(): for id_ in teuthology.roles_of_type(roles_for_host, 'mon'): - proc = run.run( - client=connections[idx], + proc = remote.run( args=[ '/tmp/cephtest/daemon-helper', '/tmp/cephtest/binary/usr/local/bin/cmon', @@ -250,10 +232,10 @@ if __name__ == '__main__': mon_daemons[id_] = proc log.info('Setting up osd nodes...') - for idx, roles_for_host in enumerate(ROLES): + osds = ctx.cluster.only(teuthology.is_type('osd')) + for remote, roles_for_host in osds.remotes.iteritems(): for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): - run.run( - client=connections[idx], + remote.run( args=[ '/tmp/cephtest/binary/usr/local/bin/cauthtool', '--create-keyring', @@ -264,10 +246,10 @@ if __name__ == '__main__': ) log.info('Setting up mds nodes...') - for idx, roles_for_host in enumerate(ROLES): + mdss = ctx.cluster.only(teuthology.is_type('mds')) + for remote, roles_for_host in mdss.remotes.iteritems(): for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): - run.run( - client=connections[idx], + remote.run( args=[ '/tmp/cephtest/binary/usr/local/bin/cauthtool', '--create-keyring', @@ -278,27 +260,27 @@ if __name__ == '__main__': ) log.info('Setting up client nodes...') - clients = cluster.only(teuthology.is_type('client')) - for id_ in teuthology.all_roles_of_type(ROLES, 'client'): - (rem,) = cluster.only('client.{id}'.format(id=id_)).remotes.keys() - rem.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/cauthtool', - '--create-keyring', - '--gen-key', - # TODO this --name= is not really obeyed, all unknown "types" are munged to "client" - '--name=client.{id}'.format(id=id_), - '/tmp/cephtest/data/client.{id}.keyring'.format(id=id_), - ], - ) + clients = ctx.cluster.only(teuthology.is_type('client')) + for remote, roles_for_host in clients.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'client'): + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/cauthtool', + '--create-keyring', + '--gen-key', + # TODO this --name= is not really obeyed, all unknown "types" are munged to "client" + '--name=client.{id}'.format(id=id_), + '/tmp/cephtest/data/client.{id}.keyring'.format(id=id_), + ], + ) log.info('Reading keys from all nodes...') keys = [] - for idx, roles_for_host in enumerate(ROLES): + for remote, roles_for_host in ctx.cluster.remotes.iteritems(): for type_ in ['osd','mds','client']: for id_ in teuthology.roles_of_type(roles_for_host, type_): data = teuthology.get_file( - remote=remotes[idx], + remote=remote, path='/tmp/cephtest/data/{type}.{id}.keyring'.format( type=type_, id=id_, @@ -348,35 +330,34 @@ if __name__ == '__main__': 'mds', 'set_max_mds', '{num_mds:d}'.format( - num_mds=teuthology.num_instances_of_type(ROLES, 'mds'), + num_mds=teuthology.num_instances_of_type(ctx.cluster, 'mds'), ), ], ) log.info('Running mkfs on osd nodes...') - for id_ in teuthology.all_roles_of_type(ROLES, 'osd'): - (rem,) = cluster.only('osd.{id}'.format(id=id_)).remotes.keys() - rem.run( - args=[ - 'mkdir', - os.path.join('/tmp/cephtest/data', 'osd.{id}.data'.format(id=id_)), - ], - ) - rem.run( - args=[ - '/tmp/cephtest/binary/usr/local/bin/cosd', - '--mkfs', - '-i', id_, - '-c', '/tmp/cephtest/ceph.conf' - ], - ) + for remote, roles_for_host in osds.remotes.iteritems(): + for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): + remote.run( + args=[ + 'mkdir', + os.path.join('/tmp/cephtest/data', 'osd.{id}.data'.format(id=id_)), + ], + ) + remote.run( + args=[ + '/tmp/cephtest/binary/usr/local/bin/cosd', + '--mkfs', + '-i', id_, + '-c', '/tmp/cephtest/ceph.conf' + ], + ) osd_daemons = {} log.info('Starting osd daemons...') - for idx, roles_for_host in enumerate(ROLES): + for remote, roles_for_host in osds.remotes.iteritems(): for id_ in teuthology.roles_of_type(roles_for_host, 'osd'): - proc = run.run( - client=connections[idx], + proc = remote.run( args=[ '/tmp/cephtest/daemon-helper', '/tmp/cephtest/binary/usr/local/bin/cosd', @@ -392,10 +373,9 @@ if __name__ == '__main__': mds_daemons = {} log.info('Starting mds daemons...') - for idx, roles_for_host in enumerate(ROLES): + for remote, roles_for_host in mdss.remotes.iteritems(): for id_ in teuthology.roles_of_type(roles_for_host, 'mds'): - proc = run.run( - client=connections[idx], + proc = remote.run( args=[ '/tmp/cephtest/daemon-helper', '/tmp/cephtest/binary/usr/local/bin/cmds', @@ -415,23 +395,20 @@ if __name__ == '__main__': remote=mon0_remote, ) - # TODO kclient mount/umount - - # TODO rbd - - run_tasks(tasks=config['tasks'], ctx=ctx) - - log.info('Shutting down mds daemons...') - for id_, proc in mds_daemons.iteritems(): - proc.stdin.close() - run.wait(mds_daemons.itervalues()) - - log.info('Shutting down osd daemons...') - for id_, proc in osd_daemons.iteritems(): - proc.stdin.close() - run.wait(osd_daemons.itervalues()) - - log.info('Shutting down mon daemons...') - for id_, proc in mon_daemons.iteritems(): - proc.stdin.close() - run.wait(mon_daemons.itervalues()) + try: + yield + finally: + log.info('Shutting down mds daemons...') + for id_, proc in mds_daemons.iteritems(): + proc.stdin.close() + run.wait(mds_daemons.itervalues()) + + log.info('Shutting down osd daemons...') + for id_, proc in osd_daemons.iteritems(): + proc.stdin.close() + run.wait(osd_daemons.itervalues()) + + log.info('Shutting down mon daemons...') + for id_, proc in mon_daemons.iteritems(): + proc.stdin.close() + run.wait(mon_daemons.itervalues()) diff --git a/daemon-helper b/teuthology/task/daemon-helper similarity index 100% rename from daemon-helper rename to teuthology/task/daemon-helper -- 2.39.5