kernel: push a local .deb instead of using gitbuilder

This lets you specify a path to an existing kernel deb
to be pushed and installed on the remote node.

Limitations:
 - We don't build the deb for you.  figuring out what
   filename 'make deb-pkg' is building is annoying.
 - We need to be able to figure out the sha1 from the provided
   path.  It shouldn't be a problem, given the way make deb-pkg
   names the debs.

diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py
index 051ce20c46240205253f5ed6f2b3b024502677a8..156a6efde585d52341739f250be2c90f22587aaa 100644 (file)
--- a/teuthology/task/kernel.py
+++ b/teuthology/task/kernel.py
@@ -1,6 +1,7 @@
 from cStringIO import StringIO
 
 import logging
+import re
 
 from teuthology import misc as teuthology
 from ..orchestra import run
@@ -23,6 +24,8 @@ def normalize_config(ctx, config):
          osd.1:
            branch: new_btrfs
            kdb: false
+         osd.3:
+           deb: /path/to/linux-whatever.deb
 
     is transformed into::
 
@@ -35,12 +38,15 @@ def normalize_config(ctx, config):
          osd.2:
            tag: v3.0
            kdb: true
+         osd.3:
+           deb: /path/to/linux-whatever.deb
 
     If config is None or just specifies a version to use,
     it is applied to all nodes.
     """
     if config is None or \
-            len(filter(lambda x: x in ['tag', 'branch', 'sha1', 'kdb'],
+            len(filter(lambda x: x in ['tag', 'branch', 'sha1', 'kdb',
+                                       'deb'],
                        config.keys())) == len(config.keys()):
         new_config = {}
         if config is None:
@@ -78,7 +84,9 @@ def validate_config(ctx, config):
 
 def need_to_install(ctx, role, sha1):
     ret = True
-    log.info('Checking kernel version of {role}...'.format(role=role))
+    log.info('Checking kernel version of {role}, want {sha1}...'.format(
+            role=role,
+            sha1=sha1))
     version_fp = StringIO()
     ctx.cluster.only(role).run(
         args=[
@@ -88,12 +96,17 @@ def need_to_install(ctx, role, sha1):
         stdout=version_fp,
         )
     version = version_fp.getvalue().rstrip('\n')
-    log.debug('current kernel version is: {version}'.format(version=version))
     if '-g' in version:
         _, current_sha1 = version.rsplit('-g', 1)
+        log.debug('current kernel version is: {version} sha1 {sha1}'.format(
+                version=version,
+                sha1=current_sha1))
         if sha1.startswith(current_sha1):
             log.debug('current sha1 is the same, do not need to install')
             ret = False
+    else:
+        log.debug('current kernel version is: {version}, unknown sha1'.format(
+                version=version))
     version_fp.close()
     return ret
 
@@ -143,34 +156,69 @@ def install_firmware(ctx, config):
                 ],
             )
 
+def download_deb(ctx, config):
+    procs = {}
+    for role, src in config.iteritems():
+        (role_remote,) = ctx.cluster.only(role).remotes.keys()
+        if src.find('/') >= 0:
+            # local deb
+            log.info('Copying kernel deb {path} to {role}...'.format(path=src,
+                                                                     role=role))
+            f = open(src, 'r')
+            proc = role_remote.run(
+                args=[
+                    'python', '-c',
+                    'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))',
+                    '/tmp/linux-image.deb',
+                    ],
+                wait=False,
+                stdin=f
+                )
+            procs[role_remote.name] = proc
+
+        else:
+            log.info('Downloading kernel {sha1} on {role}...'.format(sha1=src,
+                                                                     role=role))
+            _, deb_url = teuthology.get_ceph_binary_url(
+                package='kernel',
+                sha1=src,
+                format='deb',
+                flavor='basic',
+                arch='x86_64',
+                dist='precise',
+                )
+
+            log.info('fetching kernel from {url}'.format(url=deb_url))
+            proc = role_remote.run(
+                args=[
+                    'sudo', 'rm', '-f', '/tmp/linux-image.deb',
+                    run.Raw('&&'),
+                    'echo',
+                    'linux-image.deb',
+                    run.Raw('|'),
+                    'wget',
+                    '-nv',
+                    '-O',
+                    '/tmp/linux-image.deb',
+                    '--base={url}'.format(url=deb_url),
+                    '--input-file=-',
+                    ],
+                wait=False)
+            procs[role_remote.name] = proc
+
+    for name, proc in procs.iteritems():
+        log.debug('Waiting for download/copy to %s to complete...', name)
+        proc.exitstatus.get()
+
 
 def install_and_reboot(ctx, config):
     procs = {}
-    for role, sha1 in config.iteritems():
-        log.info('Installing kernel version {sha1} on {role}...'.format(sha1=sha1,
-                                                                        role=role))
+    for role, src in config.iteritems():
+        log.info('Installing kernel {src} on {role}...'.format(src=src,
+                                                               role=role))
         (role_remote,) = ctx.cluster.only(role).remotes.keys()
-        _, deb_url = teuthology.get_ceph_binary_url(
-            package='kernel',
-            sha1=sha1, 
-            format='deb',
-            flavor='basic',
-            arch='x86_64',
-            dist='precise',
-            )
-        log.info('fetching kernel from {url}'.format(url=deb_url))
         proc = role_remote.run(
             args=[
-                'echo',
-                'linux-image.deb',
-                run.Raw('|'),
-                'wget',
-                '-nv',
-                '-O',
-                '/tmp/linux-image.deb',
-                '--base={url}'.format(url=deb_url),
-                '--input-file=-',
-                run.Raw('&&'),
                 'sudo',
                 'dpkg',
                 '-i',
@@ -336,24 +384,40 @@ def task(ctx, config):
 
     config = normalize_config(ctx, config)
     validate_config(ctx, config)
+    log.info('config %s' % config)
 
-    need_install = {}
+    need_install = {}  # sha1 to dl, or path to deb
+    need_sha1 = {}     # sha1
     kdb = {}
     for role, role_config in config.iteritems():
-        sha1, _ = teuthology.get_ceph_binary_url(
-            package='kernel',
-            branch=role_config.get('branch'),
-            tag=role_config.get('tag'),
-            sha1=role_config.get('sha1'),
-            flavor='basic',
-            format='deb',
-            dist='precise',
-            arch='x86_64',
-            )
-        log.debug('sha1 for {role} is {sha1}'.format(role=role, sha1=sha1))
-        ctx.summary['{role}-kernel-sha1'.format(role=role)] = sha1
-        if need_to_install(ctx, role, sha1):
-            need_install[role] = sha1
+        if role_config.get('deb'):
+            path = role_config.get('deb')
+            match = re.search('\d+-g(\w{7})', path)
+            if match:
+                sha1 = match.group(1)
+                log.info('kernel deb sha1 appears to be %s', sha1)
+                if need_to_install(ctx, role, sha1):
+                    need_install[role] = path
+                    need_sha1[role] = sha1
+            else:
+                log.info('unable to extract sha1 from deb path, forcing install')
+                assert False
+        else:
+            sha1, _ = teuthology.get_ceph_binary_url(
+                package='kernel',
+                branch=role_config.get('branch'),
+                tag=role_config.get('tag'),
+                sha1=role_config.get('sha1'),
+                flavor='basic',
+                format='deb',
+                dist='precise',
+                arch='x86_64',
+                )
+            log.debug('sha1 for {role} is {sha1}'.format(role=role, sha1=sha1))
+            ctx.summary['{role}-kernel-sha1'.format(role=role)] = sha1
+            if need_to_install(ctx, role, sha1):
+                need_install[role] = sha1
+                need_sha1[role] = sha1
 
         # enable or disable kdb if specified, otherwise do not touch
         if role_config.get('kdb') is not None:
@@ -361,7 +425,8 @@ def task(ctx, config):
 
     if need_install:
         install_firmware(ctx, need_install)
+        download_deb(ctx, need_install)
         install_and_reboot(ctx, need_install)
-        wait_for_reboot(ctx, need_install, timeout)
+        wait_for_reboot(ctx, need_sha1, timeout)
 
     enable_disable_kdb(ctx, kdb)