From e4e239e2682f07bd4db83a0b3c2f627b85f28164 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sun, 29 Jul 2012 12:40:06 -0700 Subject: [PATCH] kernel: push a local .deb instead of using gitbuilder This lets you specify a path to an existing kernel deb to be pushed and installed on the remote node. Limitations: - We don't build the deb for you. figuring out what filename 'make deb-pkg' is building is annoying. - We need to be able to figure out the sha1 from the provided path. It shouldn't be a problem, given the way make deb-pkg names the debs. --- teuthology/task/kernel.py | 147 +++++++++++++++++++++++++++----------- 1 file changed, 106 insertions(+), 41 deletions(-) diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index 051ce20c46..156a6efde5 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -1,6 +1,7 @@ from cStringIO import StringIO import logging +import re from teuthology import misc as teuthology from ..orchestra import run @@ -23,6 +24,8 @@ def normalize_config(ctx, config): osd.1: branch: new_btrfs kdb: false + osd.3: + deb: /path/to/linux-whatever.deb is transformed into:: @@ -35,12 +38,15 @@ def normalize_config(ctx, config): osd.2: tag: v3.0 kdb: true + osd.3: + deb: /path/to/linux-whatever.deb If config is None or just specifies a version to use, it is applied to all nodes. """ if config is None or \ - len(filter(lambda x: x in ['tag', 'branch', 'sha1', 'kdb'], + len(filter(lambda x: x in ['tag', 'branch', 'sha1', 'kdb', + 'deb'], config.keys())) == len(config.keys()): new_config = {} if config is None: @@ -78,7 +84,9 @@ def validate_config(ctx, config): def need_to_install(ctx, role, sha1): ret = True - log.info('Checking kernel version of {role}...'.format(role=role)) + log.info('Checking kernel version of {role}, want {sha1}...'.format( + role=role, + sha1=sha1)) version_fp = StringIO() ctx.cluster.only(role).run( args=[ @@ -88,12 +96,17 @@ def need_to_install(ctx, role, sha1): stdout=version_fp, ) version = version_fp.getvalue().rstrip('\n') - log.debug('current kernel version is: {version}'.format(version=version)) if '-g' in version: _, current_sha1 = version.rsplit('-g', 1) + log.debug('current kernel version is: {version} sha1 {sha1}'.format( + version=version, + sha1=current_sha1)) if sha1.startswith(current_sha1): log.debug('current sha1 is the same, do not need to install') ret = False + else: + log.debug('current kernel version is: {version}, unknown sha1'.format( + version=version)) version_fp.close() return ret @@ -143,34 +156,69 @@ def install_firmware(ctx, config): ], ) +def download_deb(ctx, config): + procs = {} + for role, src in config.iteritems(): + (role_remote,) = ctx.cluster.only(role).remotes.keys() + if src.find('/') >= 0: + # local deb + log.info('Copying kernel deb {path} to {role}...'.format(path=src, + role=role)) + f = open(src, 'r') + proc = role_remote.run( + args=[ + 'python', '-c', + 'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))', + '/tmp/linux-image.deb', + ], + wait=False, + stdin=f + ) + procs[role_remote.name] = proc + + else: + log.info('Downloading kernel {sha1} on {role}...'.format(sha1=src, + role=role)) + _, deb_url = teuthology.get_ceph_binary_url( + package='kernel', + sha1=src, + format='deb', + flavor='basic', + arch='x86_64', + dist='precise', + ) + + log.info('fetching kernel from {url}'.format(url=deb_url)) + proc = role_remote.run( + args=[ + 'sudo', 'rm', '-f', '/tmp/linux-image.deb', + run.Raw('&&'), + 'echo', + 'linux-image.deb', + run.Raw('|'), + 'wget', + '-nv', + '-O', + '/tmp/linux-image.deb', + '--base={url}'.format(url=deb_url), + '--input-file=-', + ], + wait=False) + procs[role_remote.name] = proc + + for name, proc in procs.iteritems(): + log.debug('Waiting for download/copy to %s to complete...', name) + proc.exitstatus.get() + def install_and_reboot(ctx, config): procs = {} - for role, sha1 in config.iteritems(): - log.info('Installing kernel version {sha1} on {role}...'.format(sha1=sha1, - role=role)) + for role, src in config.iteritems(): + log.info('Installing kernel {src} on {role}...'.format(src=src, + role=role)) (role_remote,) = ctx.cluster.only(role).remotes.keys() - _, deb_url = teuthology.get_ceph_binary_url( - package='kernel', - sha1=sha1, - format='deb', - flavor='basic', - arch='x86_64', - dist='precise', - ) - log.info('fetching kernel from {url}'.format(url=deb_url)) proc = role_remote.run( args=[ - 'echo', - 'linux-image.deb', - run.Raw('|'), - 'wget', - '-nv', - '-O', - '/tmp/linux-image.deb', - '--base={url}'.format(url=deb_url), - '--input-file=-', - run.Raw('&&'), 'sudo', 'dpkg', '-i', @@ -336,24 +384,40 @@ def task(ctx, config): config = normalize_config(ctx, config) validate_config(ctx, config) + log.info('config %s' % config) - need_install = {} + need_install = {} # sha1 to dl, or path to deb + need_sha1 = {} # sha1 kdb = {} for role, role_config in config.iteritems(): - sha1, _ = teuthology.get_ceph_binary_url( - package='kernel', - branch=role_config.get('branch'), - tag=role_config.get('tag'), - sha1=role_config.get('sha1'), - flavor='basic', - format='deb', - dist='precise', - arch='x86_64', - ) - log.debug('sha1 for {role} is {sha1}'.format(role=role, sha1=sha1)) - ctx.summary['{role}-kernel-sha1'.format(role=role)] = sha1 - if need_to_install(ctx, role, sha1): - need_install[role] = sha1 + if role_config.get('deb'): + path = role_config.get('deb') + match = re.search('\d+-g(\w{7})', path) + if match: + sha1 = match.group(1) + log.info('kernel deb sha1 appears to be %s', sha1) + if need_to_install(ctx, role, sha1): + need_install[role] = path + need_sha1[role] = sha1 + else: + log.info('unable to extract sha1 from deb path, forcing install') + assert False + else: + sha1, _ = teuthology.get_ceph_binary_url( + package='kernel', + branch=role_config.get('branch'), + tag=role_config.get('tag'), + sha1=role_config.get('sha1'), + flavor='basic', + format='deb', + dist='precise', + arch='x86_64', + ) + log.debug('sha1 for {role} is {sha1}'.format(role=role, sha1=sha1)) + ctx.summary['{role}-kernel-sha1'.format(role=role)] = sha1 + if need_to_install(ctx, role, sha1): + need_install[role] = sha1 + need_sha1[role] = sha1 # enable or disable kdb if specified, otherwise do not touch if role_config.get('kdb') is not None: @@ -361,7 +425,8 @@ def task(ctx, config): if need_install: install_firmware(ctx, need_install) + download_deb(ctx, need_install) install_and_reboot(ctx, need_install) - wait_for_reboot(ctx, need_install, timeout) + wait_for_reboot(ctx, need_sha1, timeout) enable_disable_kdb(ctx, kdb) -- 2.39.5