From 3f7c9bcaa4c8fec6f1ea2326f0943a431b907d66 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 18 Feb 2013 15:06:52 -0800 Subject: [PATCH] move the install to a separate task. Signed-off-by: Sage Weil --- teuthology/nuke.py | 6 +- teuthology/task/ceph-fuse.py | 6 +- teuthology/task/ceph.py | 185 ----------------------------- teuthology/task/install.py | 222 +++++++++++++++++++++++++++++++++++ 4 files changed, 228 insertions(+), 191 deletions(-) create mode 100644 teuthology/task/install.py diff --git a/teuthology/nuke.py b/teuthology/nuke.py index 934cdedfefb5e..0c6b8a5e04f04 100644 --- a/teuthology/nuke.py +++ b/teuthology/nuke.py @@ -246,7 +246,7 @@ def dpkg_configure(ctx, log): proc.exitstatus.get() def remove_installed_packages(ctx, log): - from teuthology.task import ceph as ceph_task + from teuthology.task import install as install_task dpkg_configure(ctx, log) debs = ['ceph', @@ -256,8 +256,8 @@ def remove_installed_packages(ctx, log): 'librados2', 'librbd1', ] - ceph_task.remove_debs(ctx, debs) - ceph_task.remove_sources(ctx) + install_task.remove_debs(ctx, debs) + install_task.remove_sources(ctx) def remove_testing_tree(ctx, log): from teuthology.misc import get_testdir_base diff --git a/teuthology/task/ceph-fuse.py b/teuthology/task/ceph-fuse.py index e9900306251d4..f348a852d51ad 100644 --- a/teuthology/task/ceph-fuse.py +++ b/teuthology/task/ceph-fuse.py @@ -4,7 +4,7 @@ import os from teuthology import misc as teuthology from ..orchestra import run -from teuthology.task import ceph as ceph_task +from teuthology.task import install as install_task log = logging.getLogger(__name__) @@ -61,7 +61,7 @@ def task(ctx, config): for id_, remote in clients: # install ceph fuse package - ceph_task.install_debs(ctx, ['ceph-fuse'], config.get('branch', 'master')) + install_task.install_debs(ctx, ['ceph-fuse'], config.get('branch', 'master')) mnt = os.path.join(testdir, 'mnt.{id}'.format(id=id_)) log.info('Mounting ceph-fuse client.{id} at {remote} {mnt}...'.format( @@ -178,4 +178,4 @@ def task(ctx, config): ) # remove ceph-fuse package - ceph_task.remove_debs(ctx, ['ceph-fuse']) + install_task.remove_debs(ctx, ['ceph-fuse']) diff --git a/teuthology/task/ceph.py b/teuthology/task/ceph.py index 2bc4426725083..92ee1fa5b0f8c 100644 --- a/teuthology/task/ceph.py +++ b/teuthology/task/ceph.py @@ -145,153 +145,6 @@ def ship_utilities(ctx, config): ), ) -def _update_deb_package_list_and_install(ctx, remote, debs, branch): - """ - updates the package list so that apt-get can - download the appropriate packages - """ - - # check for ceph release key - r = remote.run( - args=[ - 'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph', - ], - stdout=StringIO(), - ) - if r.stdout.getvalue().find('Ceph automated package') == -1: - # if it doesn't exist, add it - remote.run( - args=[ - 'wget', '-q', '-O-', - 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc', - run.Raw('|'), - 'sudo', 'apt-key', 'add', '-', - ], - stdout=StringIO(), - ) - - # get ubuntu release (precise, quantal, etc.) - r = remote.run( - args=['lsb_release', '-sc'], - stdout=StringIO(), - ) - - out = r.stdout.getvalue().strip() - log.info("release type:" + out) - - gitbuilder_host = ctx.teuthology_config.get('gitbuilder_host', 'gitbuilder.ceph.com') - - # get package version string - r = remote.run( - args=[ - 'wget', '-q', '-O-', - 'http://{host}/ceph-deb-'.format(host=gitbuilder_host) + out + '-x86_64-basic/ref/' + branch + '/version', - ], - stdout=StringIO(), - ) - version = r.stdout.getvalue().strip() - log.info('package version is %s', version) - - remote.run( - args=[ - 'echo', 'deb', - 'http://{host}/ceph-deb-'.format(host=gitbuilder_host) + out + '-x86_64-basic/ref/' + branch, - out, 'main', run.Raw('|'), - 'sudo', 'tee', '/etc/apt/sources.list.d/ceph.list' - ], - stdout=StringIO(), - ) - remote.run( - args=[ - 'sudo', 'apt-get', 'update', run.Raw('&&'), - 'sudo', 'apt-get', '-y', '--force-yes', - 'install', - ] + ['%s=%s' % (d, version) for d in debs], - stdout=StringIO(), - ) - -def install_debs(ctx, debs, branch): - """ - installs Debian packages. - """ - log.info("Installing ceph debian packages: {debs}".format(debs=', '.join(debs))) - with parallel() as p: - for remote in ctx.cluster.remotes.iterkeys(): - p.spawn(_update_deb_package_list_and_install, ctx, remote, debs, branch) - -def _remove_deb(remote, debs): - args=[ - 'sudo', 'apt-get', '-y', '--force-yes', 'purge', - ] - args.extend(debs) - args.extend([ - run.Raw('||'), - 'true' - ]) - remote.run(args=args) - remote.run( - args=[ - 'sudo', 'apt-get', '-y', '--force-yes', - 'autoremove', - ], - stdout=StringIO(), - ) - -def remove_debs(ctx, debs): - log.info("Removing/purging debian packages {debs}".format(debs=', '.join(debs))) - with parallel() as p: - for remote in ctx.cluster.remotes.iterkeys(): - p.spawn(_remove_deb, remote, debs) - -def _remove_sources_list(remote): - remote.run( - args=[ - 'sudo', 'rm', '-f', '/etc/apt/sources.list.d/ceph.list', run.Raw('&&'), - 'sudo', 'apt-get', 'update', - ], - stdout=StringIO(), - ) - -def remove_sources(ctx): - log.info("Removing ceph sources list from apt") - with parallel() as p: - for remote in ctx.cluster.remotes.iterkeys(): - p.spawn(_remove_sources_list, remote) - - -@contextlib.contextmanager -def binaries(ctx, config): - - debs = [ - 'ceph', - 'ceph-dbg', - 'ceph-mds', - 'ceph-mds-dbg', - 'ceph-common', - 'ceph-common-dbg', - 'ceph-test', - 'ceph-test-dbg', - 'radosgw', - 'radosgw-dbg', - 'python-ceph', - ] - # install lib deps (so we explicitly specify version), but do not - # uninstall them, as - debs_install = debs + [ - 'librados2', - 'librados2-dbg', - 'librbd1', - 'librbd1-dbg', - ] - branch = config.get('branch', 'master') - log.info('branch: {b}'.format(b=branch)) - install_debs(ctx, debs_install, branch) - try: - yield - finally: - remove_debs(ctx, debs) - remove_sources(ctx) - def assign_devs(roles, devs): return dict(zip(roles, devs)) @@ -1155,35 +1008,6 @@ def task(ctx, config): ctx.daemons = CephState() - # Flavor tells us what gitbuilder to fetch the prebuilt software - # from. It's a combination of possible keywords, in a specific - # order, joined by dashes. It is used as a URL path name. If a - # match is not found, the teuthology run fails. This is ugly, - # and should be cleaned up at some point. - - dist = 'precise' - format = 'tarball' - arch = 'x86_64' - flavor = 'basic' - - # First element: controlled by user (or not there, by default): - # used to choose the right distribution, e.g. "oneiric". - flavor = config.get('flavor', 'basic') - - if config.get('path'): - # local dir precludes any other flavors - flavor = 'local' - else: - if config.get('valgrind'): - log.info('Using notcmalloc flavor and running some daemons under valgrind') - flavor = 'notcmalloc' - else: - if config.get('coverage'): - log.info('Recording coverage for this run.') - flavor = 'gcov' - - ctx.summary['flavor'] = flavor - testdir = teuthology.get_testdir(ctx) if config.get('coverage'): coverage_dir = '{tdir}/archive/coverage'.format(tdir=testdir) @@ -1201,15 +1025,6 @@ def task(ctx, config): with contextutil.nested( lambda: ceph_log(ctx=ctx, config=None), lambda: ship_utilities(ctx=ctx, config=None), - lambda: binaries(ctx=ctx, config=dict( - branch=config.get('branch', 'master'), - tag=config.get('tag'), - sha1=config.get('sha1'), - flavor=flavor, - dist=config.get('dist', dist), - format=format, - arch=arch - )), lambda: valgrind_post(ctx=ctx, config=config), lambda: cluster(ctx=ctx, config=dict( conf=config.get('conf', {}), diff --git a/teuthology/task/install.py b/teuthology/task/install.py new file mode 100644 index 0000000000000..3a3c6aca04255 --- /dev/null +++ b/teuthology/task/install.py @@ -0,0 +1,222 @@ +from cStringIO import StringIO + +import argparse +import contextlib +import logging +import os +import sys + +from teuthology import misc as teuthology +from teuthology import contextutil +from teuthology.parallel import parallel +from ..orchestra import run + +log = logging.getLogger(__name__) + + +def _update_deb_package_list_and_install(ctx, remote, debs, branch): + """ + updates the package list so that apt-get can + download the appropriate packages + """ + + # check for ceph release key + r = remote.run( + args=[ + 'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph', + ], + stdout=StringIO(), + ) + if r.stdout.getvalue().find('Ceph automated package') == -1: + # if it doesn't exist, add it + remote.run( + args=[ + 'wget', '-q', '-O-', + 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc', + run.Raw('|'), + 'sudo', 'apt-key', 'add', '-', + ], + stdout=StringIO(), + ) + + # get ubuntu release (precise, quantal, etc.) + r = remote.run( + args=['lsb_release', '-sc'], + stdout=StringIO(), + ) + + out = r.stdout.getvalue().strip() + log.info("release type:" + out) + + gitbuilder_host = ctx.teuthology_config.get('gitbuilder_host', + 'gitbuilder.ceph.com') + + # get package version string + r = remote.run( + args=[ + 'wget', '-q', '-O-', + 'http://{host}/ceph-deb-'.format(host=gitbuilder_host) + + out + '-x86_64-basic/ref/' + branch + '/version', + ], + stdout=StringIO(), + ) + version = r.stdout.getvalue().strip() + log.info('package version is %s', version) + + remote.run( + args=[ + 'echo', 'deb', + 'http://{host}/ceph-deb-'.format(host=gitbuilder_host) + + out + '-x86_64-basic/ref/' + branch, + out, 'main', run.Raw('|'), + 'sudo', 'tee', '/etc/apt/sources.list.d/ceph.list' + ], + stdout=StringIO(), + ) + remote.run( + args=[ + 'sudo', 'apt-get', 'update', run.Raw('&&'), + 'sudo', 'apt-get', '-y', '--force-yes', + 'install', + ] + ['%s=%s' % (d, version) for d in debs], + stdout=StringIO(), + ) + +def install_debs(ctx, debs, branch): + """ + installs Debian packages. + """ + log.info("Installing ceph debian packages: {debs}".format( + debs=', '.join(debs))) + with parallel() as p: + for remote in ctx.cluster.remotes.iterkeys(): + p.spawn( + _update_deb_package_list_and_install, ctx, remote, debs, branch) + +def _remove_deb(remote, debs): + args=[ + 'sudo', 'apt-get', '-y', '--force-yes', 'purge', + ] + args.extend(debs) + args.extend([ + run.Raw('||'), + 'true' + ]) + remote.run(args=args) + remote.run( + args=[ + 'sudo', 'apt-get', '-y', '--force-yes', + 'autoremove', + ], + stdout=StringIO(), + ) + +def remove_debs(ctx, debs): + log.info("Removing/purging debian packages {debs}".format( + debs=', '.join(debs))) + with parallel() as p: + for remote in ctx.cluster.remotes.iterkeys(): + p.spawn(_remove_deb, remote, debs) + +def _remove_sources_list(remote): + remote.run( + args=[ + 'sudo', 'rm', '-f', '/etc/apt/sources.list.d/ceph.list', + run.Raw('&&'), + 'sudo', 'apt-get', 'update', + ], + stdout=StringIO(), + ) + +def remove_sources(ctx): + log.info("Removing ceph sources list from apt") + with parallel() as p: + for remote in ctx.cluster.remotes.iterkeys(): + p.spawn(_remove_sources_list, remote) + + +@contextlib.contextmanager +def install(ctx, config): + debs = [ + 'ceph', + 'ceph-dbg', + 'ceph-mds', + 'ceph-mds-dbg', + 'ceph-common', + 'ceph-common-dbg', + 'ceph-test', + 'ceph-test-dbg', + 'radosgw', + 'radosgw-dbg', + 'python-ceph', + ] + # install lib deps (so we explicitly specify version), but do not + # uninstall them, as + debs_install = debs + [ + 'librados2', + 'librados2-dbg', + 'librbd1', + 'librbd1-dbg', + ] + branch = config.get('branch', 'master') + log.info('branch: {b}'.format(b=branch)) + install_debs(ctx, debs_install, branch) + try: + yield + finally: + remove_debs(ctx, debs) + remove_sources(ctx) + + +@contextlib.contextmanager +def task(ctx, config): + """ + Install ceph packages + """ + if config is None: + config = {} + assert isinstance(config, dict), \ + "task ceph only supports a dictionary for configuration" + + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('ceph', {})) + + # Flavor tells us what gitbuilder to fetch the prebuilt software + # from. It's a combination of possible keywords, in a specific + # order, joined by dashes. It is used as a URL path name. If a + # match is not found, the teuthology run fails. This is ugly, + # and should be cleaned up at some point. + + dist = 'precise' + arch = 'x86_64' + flavor = 'basic' + + # First element: controlled by user (or not there, by default): + # used to choose the right distribution, e.g. "oneiric". + flavor = config.get('flavor', 'basic') + + if config.get('path'): + # local dir precludes any other flavors + flavor = 'local' + else: + if config.get('valgrind'): + log.info('Using notcmalloc flavor and running some daemons under valgrind') + flavor = 'notcmalloc' + else: + if config.get('coverage'): + log.info('Recording coverage for this run.') + flavor = 'gcov' + + ctx.summary['flavor'] = flavor + + with contextutil.nested( + lambda: install(ctx=ctx, config=dict( + branch=config.get('branch', 'master'), + tag=config.get('tag'), + sha1=config.get('sha1'), + flavor=flavor, + dist=config.get('dist', dist), + arch=arch + )), + ): + yield -- 2.39.5