From 12acfc4f3aea4ffd1b112a3a464b69de9b6d6bba Mon Sep 17 00:00:00 2001 From: Andrew Schoen Date: Fri, 23 Jan 2015 11:26:43 -0600 Subject: [PATCH] Move koji related functions into teuthology.packaging We can now use these functions in other places in teuthology. I've also made the urls for koji configurable with the teuthology config. Signed-off-by: Andrew Schoen --- teuthology/config.py | 2 + teuthology/packaging.py | 103 ++++++++++++++++++++++++++++++++++++++ teuthology/task/kernel.py | 62 ++++++----------------- 3 files changed, 120 insertions(+), 47 deletions(-) diff --git a/teuthology/config.py b/teuthology/config.py index ba478628c2..aa86e7a139 100644 --- a/teuthology/config.py +++ b/teuthology/config.py @@ -138,6 +138,8 @@ class TeuthologyConfig(YamlConfig): 'src_base_path': os.path.expanduser('~/src'), 'verify_host_keys': True, 'watchdog_interval': 120, + 'kojihub_url': 'http://koji.fedoraproject.org/kojihub', + 'kojiroot_url': 'http://kojipkgs.fedoraproject.org/packages', } def __init__(self, yaml_path=None): diff --git a/teuthology/packaging.py b/teuthology/packaging.py index c856dcbba8..c49861b374 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -1,5 +1,10 @@ import logging +import ast + +from cStringIO import StringIO + from teuthology import misc +from .config import config log = logging.getLogger(__name__) @@ -94,3 +99,101 @@ def remove_package(package, remote): log.error('remove_package: bad flavor ' + flavor + '\n') return False return remote.run(args=pkgcmd) + + +def get_koji_build_info(build_id, remote, ctx): + """ + Queries kojihub and retrieves information about + the given build_id. The package, koji, must be installed + on the remote for this command to work. + + We need a remote here because koji can only be installed + on rpm based machines and teuthology runs on Ubuntu. + + Here is an example of the build info returned: + + {'owner_name': 'kdreyer', 'package_name': 'ceph', + 'task_id': 8534149, 'completion_ts': 1421278726.1171, + 'creation_event_id': 10486804, 'creation_time': '2015-01-14 18:15:17.003134', + 'epoch': None, 'nvr': 'ceph-0.80.5-4.el7ost', 'name': 'ceph', + 'completion_time': '2015-01-14 18:38:46.1171', 'state': 1, 'version': '0.80.5', + 'volume_name': 'DEFAULT', 'release': '4.el7ost', 'creation_ts': 1421277317.00313, + 'package_id': 34590, 'id': 412677, 'volume_id': 0, 'owner_id': 2826 + } + + :param build_id: The brew build_id we want to retrieve info on. + :param remote: The remote to run the koji command on. + :param ctx: The ctx from the current run, used to provide a + failure_reason and status if the koji command fails. + :returns: A python dict containing info about the build. + """ + py_cmd = ('import koji; ' + 'hub = koji.ClientSession("{kojihub_url}"); ' + 'print hub.getBuild({build_id})') + py_cmd = py_cmd.format( + build_id=build_id, + kojihub_url=config.kojihub_url + ) + proc = remote.run( + args=[ + 'python', '-c', py_cmd + ], + stdout=StringIO(), stderr=StringIO(), check_status=False + ) + if proc.exitstatus == 0: + # returns the __repr__ of a python dict + stdout = proc.stdout.getvalue().strip() + # take the __repr__ and makes it a python dict again + build_info = ast.literal_eval(stdout) + else: + msg = "Failed to query koji for build {0}".format(build_id) + log.error(msg) + log.error("stdout: {0}".format(proc.stdout.getvalue().strip())) + log.error("stderr: {0}".format(proc.stderr.getvalue().strip())) + ctx.summary["failure_reason"] = msg + ctx.summary["status"] = "dead" + raise RuntimeError(msg) + + return build_info + + +def get_kojiroot_base_url(build_info, arch="x86_64"): + """ + Builds the base download url for kojiroot given the current + build information. + + :param build_info: A dict of koji build information, possibly + retrieved from get_koji_build_info. + :param arch: The arch you want to download rpms for. + :returns: The base_url to use when downloading rpms + from brew. + """ + base_url = "{kojiroot}/{package_name}/{ver}/{rel}/{arch}/".format( + kojiroot=config.kojiroot_url, + package_name=build_info["package_name"], + ver=build_info["version"], + rel=build_info["release"], + arch=arch, + ) + return base_url + + +def get_koji_package_name(package, build_info, arch="x86_64"): + """ + Builds the package name for a brew rpm. + + :param package: The name of the package + :param build_info: A dict of koji build information, possibly + retrieved from get_brew_build_info. + :param arch: The arch you want to download rpms for. + :returns: A string representing the file name for the + requested package in koji. + """ + pkg_name = "{name}-{ver}-{rel}.{arch}.rpm".format( + name=package, + ver=build_info["version"], + rel=build_info["release"], + arch=arch, + ) + + return pkg_name diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index 69e2d616bb..18f2b6f6a9 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -9,15 +9,21 @@ import re import shlex import urllib2 import urlparse -import ast from teuthology import misc as teuthology from ..orchestra import run from ..config import config as teuth_config from ..exceptions import UnsupportedPackageTypeError, ConfigError +from ..packaging import ( + install_package, + get_koji_build_info, + get_kojiroot_base_url, + get_koji_package_name, +) log = logging.getLogger(__name__) + def normalize_config(ctx, config): """ Returns a config whose keys are all real roles. @@ -59,7 +65,7 @@ def normalize_config(ctx, config): """ if config is None or \ len(filter(lambda x: x in ['tag', 'branch', 'sha1', 'kdb', - 'deb', 'rpm', 'brew'], + 'deb', 'rpm', 'koji'], config.keys())) == len(config.keys()): new_config = {} if config is None: @@ -262,23 +268,16 @@ def download_kernel(ctx, config): (role_remote,) = ctx.cluster.only(role).remotes.keys() if isinstance(src, dict): - # we're downloading a kernel from brew, the src dict here - # is the build_info retrieved from brew using koji + # we're downloading a kernel from koji, the src dict here + # is the build_info retrieved from koji using get_koji_build_info build_id = src["id"] log.info("Downloading kernel with build_id {build_id} on {role}...".format( build_id=build_id, role=role )) needs_download = True - baseurl = "{brewroot}/kernel/{ver}/{rel}/x86_64/".format( - brewroot="http://download.devel.redhat.com/brewroot/packages", - ver=src["version"], - rel=src["release"], - ) - pkg_name = "kernel-{ver}-{rel}.x86_64.rpm".format( - ver=src["version"], - rel=src["release"], - ) + baseurl = get_kojiroot_base_url(src) + pkg_name = get_koji_package_name("kernel", src) elif src.find('/') >= 0: # local package - src is path log.info('Copying kernel package {path} to {role}...'.format( @@ -1066,41 +1065,10 @@ def task(ctx, config): # FIXME: this install should probably happen somewhere else # but I'm not sure where, so we'll leave it here for now. - log.info("Installing koji on {0}".format(role)) - role_remote.run( - args=["sudo", "yum", "-y", "install", "koji"], - ) - - # put all this in it's own function - py_cmd = ('import koji; ' - 'hub = koji.ClientSession("{brewhub_url}"); ' - 'print hub.getBuild({build_id})') - py_cmd = py_cmd.format( - build_id=build_id, - brewhub_url="http://brewhub.devel.redhat.com/brewhub" - ) - proc = role_remote.run( - args=[ - 'python', '-c', py_cmd - ], - stdout=StringIO(), stderr=StringIO(), check_status=False - ) - if proc.exitstatus == 0: - # returns the __repr__ of a python dict - stdout = proc.stdout.getvalue().strip() - # take the __repr__ and makes it a python dict again - build_info = ast.literal_eval(stdout) - else: - # this should explode - msg = "Failed to query koji for build {0}".format(build_id) - log.error(msg) - log.error("stdout: {0}".format(proc.stdout.getvalue().strip())) - log.error("stderr: {0}".format(proc.stderr.getvalue().strip())) - ctx.summary["failure_reason"] = msg - ctx.summary["status"] = "dead" - raise RuntimeError(msg) + install_package('koji', role_remote) - # end, things in a new function + # get information about this build from koji + build_info = get_koji_build_info(build_id, role_remote, ctx) need_install[role] = build_info need_version[role] = "{ver}-{rel}.x86_64".format( -- 2.39.5