From dec2c68876ac5e17af90a2a41ed61e8b46931cbe Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 2 Nov 2016 16:17:32 -0600 Subject: [PATCH] Move repo installation to GitbuilderProject Signed-off-by: Zack Cerza --- teuthology/packaging.py | 79 ++++++++++++++++++++++++++++- teuthology/task/install/__init__.py | 10 +--- teuthology/task/install/deb.py | 36 +++---------- teuthology/task/install/rpm.py | 76 +++++---------------------- 4 files changed, 98 insertions(+), 103 deletions(-) diff --git a/teuthology/packaging.py b/teuthology/packaging.py index 1e2a67eb44..57e386b921 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -12,8 +12,10 @@ from . import repo_utils from .config import config from .contextutil import safe_while -from .exceptions import VersionNotFoundError, CommitNotFoundError +from .exceptions import (VersionNotFoundError, CommitNotFoundError, + NoRemoteError) from .orchestra.opsys import OS, DEFAULT_OS_VERSION +from .orchestra.run import Raw log = logging.getLogger(__name__) @@ -398,7 +400,7 @@ def _get_config_value_for_remote(ctx, remote, config, key): :param config: the config dict :param key: the name of the value to retrieve """ - roles = ctx.cluster.remotes[remote] + roles = ctx.cluster.remotes[remote] if ctx else None if 'all' in config: return config['all'].get(key) elif roles: @@ -436,6 +438,8 @@ class GitbuilderProject(object): """ Represents a project that is built by gitbuilder. """ + # gitbuilder always uses this value + rpm_release = "1-0" def __init__(self, project, job_config, ctx=None, remote=None): self.project = project @@ -759,6 +763,77 @@ class GitbuilderProject(object): return sha1 + def install_repo(self): + """ + Install the .repo file or sources.list fragment on self.remote if there + is one. If not, raises an exception + """ + if not self.remote: + raise NoRemoteError() + if self.remote.os.package_type == 'rpm': + self._install_rpm_repo() + elif self.remote.os.package_type == 'deb': + self._install_deb_repo() + + def _install_rpm_repo(self): + dist_release = self.dist_release + project = self.project + if dist_release == 'opensuse': + proj_release = '{proj}-release-{release}.noarch'.format( + proj=project, release=self.rpm_release) + else: + proj_release = \ + '{proj}-release-{release}.{dist_release}.noarch'.format( + proj=project, release=self.rpm_release, + dist_release=dist_release + ) + rpm_name = "{rpm_nm}.rpm".format(rpm_nm=proj_release) + url = "{base_url}/noarch/{rpm_name}".format( + base_url=self.base_url, rpm_name=rpm_name) + if dist_release == 'opensuse': + self.remote.run(args=[ + 'sudo', 'zypper', '-n', 'install', '--capability', rpm_name + ]) + else: + self.remote.run(args=['sudo', 'yum', '-y', 'install', url]) + + def _install_deb_repo(self): + self.remote.run( + args=[ + 'echo', 'deb', self.base_url, self.codename, 'main', + Raw('|'), + 'sudo', 'tee', + '/etc/apt/sources.list.d/{proj}.list'.format( + proj=self.project), + ], + stdout=StringIO(), + ) + + def remove_repo(self): + """ + Remove the .repo file or sources.list fragment on self.remote if there + is one. If not, raises an exception + """ + if not self.remote: + raise NoRemoteError() + if self.remote.os.package_type == 'rpm': + self._remove_rpm_repo() + elif self.remote.os.package_type == 'deb': + self._remove_deb_repo() + + def _remove_rpm_repo(self): + remove_package('%s-release' % self.project, self.remote) + + def _remove_deb_repo(self): + self.remote.run( + args=[ + 'sudo', + 'rm', '-f', + '/etc/apt/sources.list.d/{proj}.list'.format( + proj=self.project), + ] + ) + class ShamanProject(GitbuilderProject): def __init__(self, project, job_config, ctx=None, remote=None): diff --git a/teuthology/task/install/__init__.py b/teuthology/task/install/__init__.py index be109f0e67..44d926e769 100644 --- a/teuthology/task/install/__init__.py +++ b/teuthology/task/install/__init__.py @@ -154,15 +154,7 @@ def remove_sources(ctx, config): proj=project)) for remote in ctx.cluster.remotes.iterkeys(): remove_fn = remove_sources_pkgs[remote.os.package_type] - p.spawn(remove_fn, remote, project) - - with parallel() as p: - project = 'calamari' - log.info("Removing {proj} sources lists".format( - proj=project)) - for remote in ctx.cluster.remotes.iterkeys(): - remove_fn = remove_sources_pkgs[remote.os.package_type] - p.spawn(remove_fn, remote, project) + p.spawn(remove_fn, ctx, config, remote) def get_package_list(ctx, config): diff --git a/teuthology/task/install/deb.py b/teuthology/task/install/deb.py index 6135979e13..6ce9b69cc3 100644 --- a/teuthology/task/install/deb.py +++ b/teuthology/task/install/deb.py @@ -54,15 +54,8 @@ def _update_package_list_and_install(ctx, remote, debs, config): version = builder.version log.info('Package version is %s', version) - remote.run( - args=[ - 'echo', 'deb', builder.base_url, builder.codename, 'main', - run.Raw('|'), - 'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format( - proj=config.get('project', 'ceph')), - ], - stdout=StringIO(), - ) + builder.install_repo() + remote.run(args=['sudo', 'apt-get', 'update'], check_status=False) remote.run( args=[ @@ -137,19 +130,11 @@ def _remove(ctx, config, remote, debs): ) -def _remove_sources_list(remote, proj): - """ - Removes /etc/apt/sources.list.d/{proj}.list and then runs ``apt-get - update``. - - :param remote: the teuthology.orchestra.remote.Remote object - :param proj: the project whose sources.list needs removing - """ +def _remove_sources_list(ctx, config, remote): + builder = _get_builder_project(ctx, remote, config) + builder.remove_repo() remote.run( args=[ - 'sudo', 'rm', '-f', '/etc/apt/sources.list.d/{proj}.list'.format( - proj=proj), - run.Raw('&&'), 'sudo', 'apt-get', 'update', ], check_status=False, @@ -195,15 +180,8 @@ def _upgrade_packages(ctx, config, remote, debs): version = builder.version log.info('Package version is %s', version) - remote.run( - args=[ - 'echo', 'deb', base_url, builder.codename, 'main', - run.Raw('|'), - 'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format( - proj=config.get('project', 'ceph')), - ], - stdout=StringIO(), - ) + builder.install_repo() + remote.run(args=['sudo', 'apt-get', 'update'], check_status=False) remote.run( args=[ diff --git a/teuthology/task/install/rpm.py b/teuthology/task/install/rpm.py index d89e5a5bcf..b84ee6041d 100644 --- a/teuthology/task/install/rpm.py +++ b/teuthology/task/install/rpm.py @@ -8,9 +8,6 @@ from .util import _get_builder_project, _get_local_dir log = logging.getLogger(__name__) -# Should the RELEASE value get extracted from somewhere? -RELEASE = "1-0" - def _remove(ctx, config, remote, rpm): """ @@ -55,16 +52,8 @@ def _remove(ctx, config, remote, rpm): args=[ 'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts, ]) - if dist_release == 'opensuse': - projRelease = '%s-release-%s.noarch' % ( - config.get('project', 'ceph'), RELEASE) - else: - projRelease = '%s-release-%s.%s.noarch' % ( - config.get('project', 'ceph'), RELEASE, dist_release) - if dist_release == 'opensuse': - remote.run(args=['sudo', 'zypper', '-n', 'remove', projRelease]) - else: - remote.run(args=['sudo', 'yum', 'erase', projRelease, '-y']) + + builder.remove_repo() if dist_release != 'opensuse': pkg_mng_opts = 'expire-cache' @@ -94,7 +83,7 @@ def _package_overrides(pkgs, os): def _update_package_list_and_install(ctx, remote, rpm, config): """ - Installs the ceph-release package for the relevant branch, then installs + Installs the repository for the relevant branch, then installs the requested packages on the remote system. TODO: split this into at least two functions. @@ -110,25 +99,10 @@ def _update_package_list_and_install(ctx, remote, rpm, config): log.info('Package version is %s', builder.version) log.info("Installing packages: {pkglist} on remote rpm {arch}".format( pkglist=", ".join(rpm), arch=builder.arch)) + builder.install_repo() + dist_release = builder.dist_release project = builder.project - start_of_url = builder.base_url - if dist_release == 'opensuse': - proj_release = '{proj}-release-{release}.noarch'.format( - proj=project, release=RELEASE) - else: - proj_release = '{proj}-release-{release}.{dist_release}.noarch'.format( - proj=project, release=RELEASE, dist_release=dist_release) - rpm_name = "{rpm_nm}.rpm".format(rpm_nm=proj_release) - base_url = "{start_of_url}/noarch/{rpm_name}".format( - start_of_url=start_of_url, rpm_name=rpm_name) - if dist_release == 'opensuse': - remote.run(args=[ - 'sudo', 'zypper', '-n', 'install', '--capability', rpm_name - ]) - else: - remote.run(args=['sudo', 'yum', '-y', 'install', base_url]) - if dist_release != 'opensuse': uri = builder.uri_reference _yum_fix_repo_priority(remote, project, uri) @@ -264,19 +238,16 @@ def _yum_unset_check_obsoletes(remote): check_status=False) -def _remove_sources_list(remote, proj): +def _remove_sources_list(ctx, config, remote): """ Removes /etc/yum.repos.d/{proj}.repo, /var/lib/{proj}, and /var/log/{proj} :param remote: the teuthology.orchestra.remote.Remote object :param proj: the project whose .repo needs removing """ - if remote.os.name != 'opensuse': - remote.run( - args=['sudo', 'rm', - '/etc/yum.repos.d/{proj}.repo'.format(proj=proj)], - check_status=False, - ) + builder = _get_builder_project(ctx, remote, config) + builder.remove_repo() + proj = builder.project # FIXME # There probably should be a way of removing these files that is # implemented in the yum/rpm remove procedures for the ceph package. @@ -296,7 +267,7 @@ def _remove_sources_list(remote, proj): def _upgrade_packages(ctx, config, remote, pkgs): """ Upgrade project's packages on remote RPM-based host - Before doing so, it makes sure the project's -release RPM is installed - + Before doing so, it makes sure the project's repository is installed - removing any previous version first. :param ctx: the argparse.Namespace object @@ -318,30 +289,9 @@ def _upgrade_packages(ctx, config, remote, pkgs): log.info('Repo base URL: %s', base_url) project = builder.project - # Remove the -release package before upgrading it - args = ['sudo', 'rpm', '-ev', '%s-release' % project] - remote.run(args=args) - - # Build the new -release package path - if (builder.dist_release == 'opensuse'): - release_rpm = \ - "{base}/noarch/{proj}-release-{release}.noarch.rpm".format( - base=base_url, - proj=project, - release=RELEASE - ) - else: - release_rpm = \ - "{base}/noarch/{proj}-release-{release}.{dist_release}.noarch.rpm".format( - base=base_url, - proj=project, - release=RELEASE, - dist_release=builder.dist_release, - ) - - # Upgrade the -release package - args = ['sudo', 'rpm', '-Uv', release_rpm] - remote.run(args=args) + # Remove the repository before re-adding it + builder.remove_repo() + builder.install_repo() if builder.dist_release != 'opensuse': uri = builder.uri_reference -- 2.39.5