From: Andrew Schoen Date: Fri, 23 Jan 2015 17:26:43 +0000 (-0600) Subject: Move koji related functions into teuthology.packaging X-Git-Tag: 1.1.0~1028^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=12acfc4f3aea4ffd1b112a3a464b69de9b6d6bba;p=teuthology.git Move koji related functions into teuthology.packaging We can now use these functions in other places in teuthology. I've also made the urls for koji configurable with the teuthology config. Signed-off-by: Andrew Schoen --- diff --git a/teuthology/config.py b/teuthology/config.py index ba478628c..aa86e7a13 100644 --- a/teuthology/config.py +++ b/teuthology/config.py @@ -138,6 +138,8 @@ class TeuthologyConfig(YamlConfig): 'src_base_path': os.path.expanduser('~/src'), 'verify_host_keys': True, 'watchdog_interval': 120, + 'kojihub_url': 'http://koji.fedoraproject.org/kojihub', + 'kojiroot_url': 'http://kojipkgs.fedoraproject.org/packages', } def __init__(self, yaml_path=None): diff --git a/teuthology/packaging.py b/teuthology/packaging.py index c856dcbba..c49861b37 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -1,5 +1,10 @@ import logging +import ast + +from cStringIO import StringIO + from teuthology import misc +from .config import config log = logging.getLogger(__name__) @@ -94,3 +99,101 @@ def remove_package(package, remote): log.error('remove_package: bad flavor ' + flavor + '\n') return False return remote.run(args=pkgcmd) + + +def get_koji_build_info(build_id, remote, ctx): + """ + Queries kojihub and retrieves information about + the given build_id. The package, koji, must be installed + on the remote for this command to work. + + We need a remote here because koji can only be installed + on rpm based machines and teuthology runs on Ubuntu. + + Here is an example of the build info returned: + + {'owner_name': 'kdreyer', 'package_name': 'ceph', + 'task_id': 8534149, 'completion_ts': 1421278726.1171, + 'creation_event_id': 10486804, 'creation_time': '2015-01-14 18:15:17.003134', + 'epoch': None, 'nvr': 'ceph-0.80.5-4.el7ost', 'name': 'ceph', + 'completion_time': '2015-01-14 18:38:46.1171', 'state': 1, 'version': '0.80.5', + 'volume_name': 'DEFAULT', 'release': '4.el7ost', 'creation_ts': 1421277317.00313, + 'package_id': 34590, 'id': 412677, 'volume_id': 0, 'owner_id': 2826 + } + + :param build_id: The brew build_id we want to retrieve info on. + :param remote: The remote to run the koji command on. + :param ctx: The ctx from the current run, used to provide a + failure_reason and status if the koji command fails. + :returns: A python dict containing info about the build. + """ + py_cmd = ('import koji; ' + 'hub = koji.ClientSession("{kojihub_url}"); ' + 'print hub.getBuild({build_id})') + py_cmd = py_cmd.format( + build_id=build_id, + kojihub_url=config.kojihub_url + ) + proc = remote.run( + args=[ + 'python', '-c', py_cmd + ], + stdout=StringIO(), stderr=StringIO(), check_status=False + ) + if proc.exitstatus == 0: + # returns the __repr__ of a python dict + stdout = proc.stdout.getvalue().strip() + # take the __repr__ and makes it a python dict again + build_info = ast.literal_eval(stdout) + else: + msg = "Failed to query koji for build {0}".format(build_id) + log.error(msg) + log.error("stdout: {0}".format(proc.stdout.getvalue().strip())) + log.error("stderr: {0}".format(proc.stderr.getvalue().strip())) + ctx.summary["failure_reason"] = msg + ctx.summary["status"] = "dead" + raise RuntimeError(msg) + + return build_info + + +def get_kojiroot_base_url(build_info, arch="x86_64"): + """ + Builds the base download url for kojiroot given the current + build information. + + :param build_info: A dict of koji build information, possibly + retrieved from get_koji_build_info. + :param arch: The arch you want to download rpms for. + :returns: The base_url to use when downloading rpms + from brew. + """ + base_url = "{kojiroot}/{package_name}/{ver}/{rel}/{arch}/".format( + kojiroot=config.kojiroot_url, + package_name=build_info["package_name"], + ver=build_info["version"], + rel=build_info["release"], + arch=arch, + ) + return base_url + + +def get_koji_package_name(package, build_info, arch="x86_64"): + """ + Builds the package name for a brew rpm. + + :param package: The name of the package + :param build_info: A dict of koji build information, possibly + retrieved from get_brew_build_info. + :param arch: The arch you want to download rpms for. + :returns: A string representing the file name for the + requested package in koji. + """ + pkg_name = "{name}-{ver}-{rel}.{arch}.rpm".format( + name=package, + ver=build_info["version"], + rel=build_info["release"], + arch=arch, + ) + + return pkg_name diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index 69e2d616b..18f2b6f6a 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -9,15 +9,21 @@ import re import shlex import urllib2 import urlparse -import ast from teuthology import misc as teuthology from ..orchestra import run from ..config import config as teuth_config from ..exceptions import UnsupportedPackageTypeError, ConfigError +from ..packaging import ( + install_package, + get_koji_build_info, + get_kojiroot_base_url, + get_koji_package_name, +) log = logging.getLogger(__name__) + def normalize_config(ctx, config): """ Returns a config whose keys are all real roles. @@ -59,7 +65,7 @@ def normalize_config(ctx, config): """ if config is None or \ len(filter(lambda x: x in ['tag', 'branch', 'sha1', 'kdb', - 'deb', 'rpm', 'brew'], + 'deb', 'rpm', 'koji'], config.keys())) == len(config.keys()): new_config = {} if config is None: @@ -262,23 +268,16 @@ def download_kernel(ctx, config): (role_remote,) = ctx.cluster.only(role).remotes.keys() if isinstance(src, dict): - # we're downloading a kernel from brew, the src dict here - # is the build_info retrieved from brew using koji + # we're downloading a kernel from koji, the src dict here + # is the build_info retrieved from koji using get_koji_build_info build_id = src["id"] log.info("Downloading kernel with build_id {build_id} on {role}...".format( build_id=build_id, role=role )) needs_download = True - baseurl = "{brewroot}/kernel/{ver}/{rel}/x86_64/".format( - brewroot="http://download.devel.redhat.com/brewroot/packages", - ver=src["version"], - rel=src["release"], - ) - pkg_name = "kernel-{ver}-{rel}.x86_64.rpm".format( - ver=src["version"], - rel=src["release"], - ) + baseurl = get_kojiroot_base_url(src) + pkg_name = get_koji_package_name("kernel", src) elif src.find('/') >= 0: # local package - src is path log.info('Copying kernel package {path} to {role}...'.format( @@ -1066,41 +1065,10 @@ def task(ctx, config): # FIXME: this install should probably happen somewhere else # but I'm not sure where, so we'll leave it here for now. - log.info("Installing koji on {0}".format(role)) - role_remote.run( - args=["sudo", "yum", "-y", "install", "koji"], - ) - - # put all this in it's own function - py_cmd = ('import koji; ' - 'hub = koji.ClientSession("{brewhub_url}"); ' - 'print hub.getBuild({build_id})') - py_cmd = py_cmd.format( - build_id=build_id, - brewhub_url="http://brewhub.devel.redhat.com/brewhub" - ) - proc = role_remote.run( - args=[ - 'python', '-c', py_cmd - ], - stdout=StringIO(), stderr=StringIO(), check_status=False - ) - if proc.exitstatus == 0: - # returns the __repr__ of a python dict - stdout = proc.stdout.getvalue().strip() - # take the __repr__ and makes it a python dict again - build_info = ast.literal_eval(stdout) - else: - # this should explode - msg = "Failed to query koji for build {0}".format(build_id) - log.error(msg) - log.error("stdout: {0}".format(proc.stdout.getvalue().strip())) - log.error("stderr: {0}".format(proc.stderr.getvalue().strip())) - ctx.summary["failure_reason"] = msg - ctx.summary["status"] = "dead" - raise RuntimeError(msg) + install_package('koji', role_remote) - # end, things in a new function + # get information about this build from koji + build_info = get_koji_build_info(build_id, role_remote, ctx) need_install[role] = build_info need_version[role] = "{ver}-{rel}.x86_64".format(