From ff66acf18771b725ede50ddb5c3b04d166149d86 Mon Sep 17 00:00:00 2001 From: Kyr Shatskyy Date: Thu, 26 Oct 2017 18:15:41 +0200 Subject: [PATCH] openstack: add --test-repo CLI option Add custom repos before installing rpm packages on test nodes. Repository can be specified as a NAME:URL pair. Several repos can be provided by specifying the option multiple times. For example, --test-repo foo:http://example.com/repo/foo \ --test-repo bar:http://example.com/repo/bar gives two test package repositories named "foo" and "bar". Conflicts: scripts/openstack.py teuthology/openstack/__init__.py teuthology/orchestra/opsys.py teuthology/task/install/rpm.py --- scripts/openstack.py | 11 ++ teuthology/openstack/__init__.py | 39 +++++- .../{openstack.yaml => openstack-basic.yaml} | 0 .../openstack/openstack-buildpackages.yaml | 10 ++ teuthology/orchestra/opsys.py | 5 +- teuthology/packaging.py | 7 +- teuthology/run.py | 13 +- teuthology/task/install/__init__.py | 15 ++- teuthology/task/install/rpm.py | 114 +++++++++++------- teuthology/task/internal/__init__.py | 4 +- 10 files changed, 155 insertions(+), 63 deletions(-) rename teuthology/openstack/{openstack.yaml => openstack-basic.yaml} (100%) create mode 100644 teuthology/openstack/openstack-buildpackages.yaml diff --git a/scripts/openstack.py b/scripts/openstack.py index fd4168db9c..e9f9aa4315 100644 --- a/scripts/openstack.py +++ b/scripts/openstack.py @@ -227,6 +227,17 @@ def get_openstack_parser(): help='Public facing URL where archives are uploaded', default='http://teuthology-logs.public.ceph.com', ) + parser.add_argument( + '--test-repo', + action='append', + help=('Package repository, or repositories, to be added on test nodes. ' + 'Repository to be specified as a NAME:URL pair. Multiple ' + 'repositories can be provided with multiple usage. ' + 'For example --test-repo foo:http://example.com/repo/foo ' + '--test-repo bar:http://example.com/repo/bar specifies two ' + 'test package repositories named "foo" and "bar", respectively.'), + default=None, + ) return parser def get_parser(): diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py index ad89a6fced..31ee4cb232 100644 --- a/teuthology/openstack/__init__.py +++ b/teuthology/openstack/__init__.py @@ -34,6 +34,7 @@ import tempfile import teuthology import time import types +import yaml from subprocess import CalledProcessError @@ -231,8 +232,8 @@ class OpenStack(object): os.environ['OS_TOKEN_VALUE'] = OpenStack.token OpenStack.token_expires = int(time.time() + OpenStack.token_cache_duration) os.environ['OS_TOKEN_EXPIRES'] = str(OpenStack.token_expires) - log.info("caching OS_TOKEN_VALUE " - "during %s seconds" % OpenStack.token_cache_duration) + log.debug("caching OS_TOKEN_VALUE " + "during %s seconds" % OpenStack.token_cache_duration) return True def get_os_url(self, cmd, type=None): @@ -620,6 +621,7 @@ class TeuthologyOpenStack(OpenStack): if original_argv[0] in ('--name', '--teuthology-branch', '--teuthology-git-url', + '--test-repo', '--archive-upload', '--archive-upload-url', '--key-name', @@ -635,6 +637,30 @@ class TeuthologyOpenStack(OpenStack): original_argv.pop(0) else: argv.append(original_argv.pop(0)) + if self.args.test_repo: + repos = [{'name':k, 'url': v} + for k, v in [x.split(':', 1) + for x in self.args.test_repo]] + log.info("Using repos: %s" % self.args.test_repo) + + overrides = { + 'overrides': { + 'install': { + 'repos' : repos + } + } + } + yaml_data = yaml.dump(overrides, default_flow_style=False) + with tempfile.NamedTemporaryFile(mode='w+b', + suffix='-artifact.yaml', + delete=False) as f: + yaml_file = f.name + log.debug("Using file " + yaml_file) + print >> f, yaml_data + + path = self._upload_yaml_file(yaml_file) + argv.append(path) + # # If --upload, provide --archive-upload{,-url} regardless of # what was originally provided on the command line because the @@ -651,8 +677,13 @@ class TeuthologyOpenStack(OpenStack): " ~/.teuthology.yaml" ).format(opt='ceph_git_url', value=ceph_repo) self.ssh(command) - argv.append('/home/' + self.username + - '/teuthology/teuthology/openstack/openstack.yaml') + user_home = '/home/' + self.username + openstack_home = user_home + '/teuthology/teuthology/openstack' + if self.args.test_repo: + argv.append(openstack_home + '/openstack-basic.yaml') + else: + argv.append(openstack_home + '/openstack-basic.yaml') + argv.append(openstack_home + '/openstack-buildpackages.yaml') command = ( "source ~/.bashrc_teuthology ; " + self.teuthology_suite + " " + " --machine-type openstack " + diff --git a/teuthology/openstack/openstack.yaml b/teuthology/openstack/openstack-basic.yaml similarity index 100% rename from teuthology/openstack/openstack.yaml rename to teuthology/openstack/openstack-basic.yaml diff --git a/teuthology/openstack/openstack-buildpackages.yaml b/teuthology/openstack/openstack-buildpackages.yaml new file mode 100644 index 0000000000..1e404b48c6 --- /dev/null +++ b/teuthology/openstack/openstack-buildpackages.yaml @@ -0,0 +1,10 @@ +tasks: + - buildpackages: + good_machine: + disk: 100 # GB + ram: 15000 # MB + cpus: 16 + min_machine: + disk: 100 # GB + ram: 8000 # MB + cpus: 1 diff --git a/teuthology/orchestra/opsys.py b/teuthology/orchestra/opsys.py index e328ae9d61..21f71ca94f 100644 --- a/teuthology/orchestra/opsys.py +++ b/teuthology/orchestra/opsys.py @@ -44,6 +44,7 @@ DISTRO_CODENAME_MAP = { }, "sle": { "12.2": "sle", + "12.3": "sle", }, "opensuse-leap": { "42.2": "leap", @@ -57,7 +58,7 @@ DEFAULT_OS_VERSION = dict( fedora="25", centos="7.4", opensuse="42.1", - sle="12.2", + sle="12.3", rhel="7.4", debian='8.0' ) @@ -85,7 +86,7 @@ class OS(object): @staticmethod def _version_to_codename(name, version): for (_version, codename) in DISTRO_CODENAME_MAP[name].iteritems(): - if version == _version or version.split('.')[0] == _version: + if str(version) == _version or str(version).split('.')[0] == _version: return codename @staticmethod diff --git a/teuthology/packaging.py b/teuthology/packaging.py index e847fc0f03..349dbbcabc 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -641,7 +641,7 @@ class GitbuilderProject(object): if not version: version = DEFAULT_OS_VERSION.get(self.os_type) - return version + return str(version) def _get_uri_reference(self): """ @@ -796,9 +796,8 @@ class GitbuilderProject(object): url = "{base_url}/{arch}".format( base_url=self.base_url, arch=self.arch) self.remote.run(args=[ - 'sudo', 'zypper', '-n', 'addrepo', '-p', '1', url, 'ceph-rpm-under-test', - Raw(';'), - 'sudo', 'zypper', '-n', '--no-gpg-checks', 'refresh' + 'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck', + '-p', '1', url, 'ceph-rpm-under-test', ]) else: self.remote.run(args=['sudo', 'yum', '-y', 'install', url]) diff --git a/teuthology/run.py b/teuthology/run.py index 78c0e03612..f61f2a6f28 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -168,10 +168,15 @@ def validate_tasks(config): def get_initial_tasks(lock, config, machine_type): - init_tasks = [ - {'internal.check_packages': None}, - {'internal.buildpackages_prep': None}, - ] + init_tasks = [] + overrides = config.get('overrides') + having_repos = ('install' in config and 'repos' in config.get('install')) \ + or (overrides and 'install' in overrides and 'repos' in overrides.get('install')) + if not having_repos: + init_tasks += [ + {'internal.check_packages': None}, + {'internal.buildpackages_prep': None}, + ] if 'roles' in config and lock: msg = ('You cannot specify targets in a config file when using the ' + '--lock option') diff --git a/teuthology/task/install/__init__.py b/teuthology/task/install/__init__.py index d05359c150..9adc099ffc 100644 --- a/teuthology/task/install/__init__.py +++ b/teuthology/task/install/__init__.py @@ -41,6 +41,9 @@ def verify_package_version(ctx, config, remote): if config.get("extras"): log.info("Skipping version verification...") return True + if 'repos' in config and config.get('repos'): + log.info("Skipping version verification because we have custom repos...") + return True builder = _get_builder_project(ctx, remote, config) version = builder.version pkg_to_check = builder.project @@ -102,7 +105,6 @@ def _purge_data(remote): 'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph', ]) - def install_packages(ctx, pkgs, config): """ Installs packages on each remote in ctx. @@ -576,6 +578,8 @@ def task(ctx, config): if overrides: install_overrides = overrides.get('install', {}) teuthology.deep_merge(config, install_overrides.get(project, {})) + repos = install_overrides.get('repos', None) + log.debug('INSTALL overrides: %s' % install_overrides) log.debug('config %s' % config) rhbuild = None @@ -602,8 +606,7 @@ def task(ctx, config): with contextutil.nested(*nested_tasks): yield else: - with contextutil.nested( - lambda: install(ctx=ctx, config=dict( + nested_config = dict( branch=config.get('branch'), tag=config.get('tag'), sha1=config.get('sha1'), @@ -616,7 +619,11 @@ def task(ctx, config): wait_for_package=config.get('wait_for_package', False), project=project, packages=config.get('packages', dict()), - )), + ) + if repos: + nested_config['repos'] = repos + with contextutil.nested( + lambda: install(ctx=ctx, config=nested_config), lambda: ship_utilities(ctx=ctx, config=None), ): yield diff --git a/teuthology/task/install/rpm.py b/teuthology/task/install/rpm.py index cc6c8781e9..0c1a9adc3b 100644 --- a/teuthology/task/install/rpm.py +++ b/teuthology/task/install/rpm.py @@ -18,49 +18,40 @@ def _remove(ctx, config, remote, rpm): :param remote: the teuthology.orchestra.remote.Remote object :param rpm: list of packages names to remove """ - rpm = _package_overrides(rpm, remote.os) + remote_os = remote.os + dist_release = remote_os.name + rpm = _package_overrides(rpm, remote_os) log.info("Removing packages: {pkglist} on rpm system.".format( pkglist=", ".join(rpm))) - builder = _get_builder_project(ctx, remote, config) - dist_release = builder.dist_release + repos = config.get('repos') if dist_release in ['opensuse', 'sle']: - pkg_mng_cmd = 'zypper' - pkg_mng_opts = '-n --no-gpg-checks' - pkg_mng_subcommand_opts = '--capability' + remote.run(args=''' + for d in {rpms} ; do + sudo zypper -n --no-gpg-checks remove --capability $d || true + done'''.format(rpms=' '.join(rpm))) + remote.run(args='sudo zypper clean -a') else: - pkg_mng_cmd = 'yum' - pkg_mng_opts = '-y' - pkg_mng_subcommand_opts = '' + remote.run(args=''' + for d in {rpms} ; do + sudo yum -y remove $d || true + done'''.format(rpms=' '.join(rpm))) + remote.run(args='sudo yum clean all') + + if repos: + if dist_release in ['opensuse', 'sle']: + _zypper_removerepo(remote, repos) + else: + raise Exception('Custom repos were specified for %s ' % remote_os + + 'but these are currently not supported') + else: + builder = _get_builder_project(ctx, remote, config) + builder.remove_repo() - remote.run( - args=[ - 'for', 'd', 'in', - ] + rpm + [ - run.Raw(';'), - 'do', - 'sudo', - pkg_mng_cmd, pkg_mng_opts, 'remove', pkg_mng_subcommand_opts, - run.Raw('$d'), run.Raw('||'), 'true', run.Raw(';'), - 'done', - ]) if dist_release in ['opensuse', 'sle']: - pkg_mng_opts = '-a' + remote.run(args='sudo zypper clean -a') else: - pkg_mng_opts = 'all' - remote.run( - args=[ - 'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts, - ]) - - builder.remove_repo() - - if dist_release not in ['opensuse', 'sle']: - pkg_mng_opts = 'expire-cache' - remote.run( - args=[ - 'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts, - ]) + remote.run(args='sudo yum clean expire-cache') def _package_overrides(pkgs, os): @@ -80,6 +71,32 @@ def _package_overrides(pkgs, os): result.append(pkg) return result +def _zypper_addrepo(remote, repo_list): + """ + Add zypper repos to the remote system. + + :param remote: remote node where to add packages + :param repo_list: list of dictionaries with keys 'name', 'url' + :return: + """ + for repo in repo_list: + remote.run(args=[ + 'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck', + '-p', '1', repo['url'], repo['name'], + ]) + +def _zypper_removerepo(remote, repo_list): + """ + Remove zypper repos on the remote system. + + :param remote: remote node where to remove packages from + :param repo_list: list of dictionaries with keys 'name', 'url' + :return: + """ + for repo in repo_list: + remote.run(args=[ + 'sudo', 'zypper', '-n', 'removerepo', repo['name'], + ]) def _update_package_list_and_install(ctx, remote, rpm, config): """ @@ -101,17 +118,28 @@ def _update_package_list_and_install(ctx, remote, rpm, config): rpm += system_pkglist.get('rpm') else: rpm += system_pkglist - rpm = _package_overrides(rpm, remote.os) - builder = _get_builder_project(ctx, remote, config) - log.info('Pulling from %s', builder.base_url) - log.info('Package version is %s', builder.version) + remote_os = remote.os + rpm = _package_overrides(rpm, remote_os) log.info("Installing packages: {pkglist} on remote rpm {arch}".format( - pkglist=", ".join(rpm), arch=builder.arch)) - builder.install_repo() + pkglist=", ".join(rpm), arch=remote.arch)) + + dist_release = remote_os.name + repos = config.get('repos') + if repos: + log.debug("Adding repos: %s" % repos) + if dist_release in ['opensuse', 'sle']: + _zypper_addrepo(remote, repos) + else: + raise Exception('Custom repos were specified for %s ' % remote_os + + 'but these are currently not supported') + else: + builder = _get_builder_project(ctx, remote, config) + log.info('Pulling from %s', builder.base_url) + log.info('Package version is %s', builder.version) + builder.install_repo() - dist_release = builder.dist_release - project = builder.project if dist_release not in ['opensuse', 'sle']: + project = builder.project uri = builder.uri_reference _yum_fix_repo_priority(remote, project, uri) _yum_fix_repo_host(remote, project) diff --git a/teuthology/task/internal/__init__.py b/teuthology/task/internal/__init__.py index f5f678d8aa..7d775d4b66 100644 --- a/teuthology/task/internal/__init__.py +++ b/teuthology/task/internal/__init__.py @@ -82,8 +82,8 @@ def check_packages(ctx, config): # in the job config. if os_type and sha1: package = get_builder_project()("ceph", ctx.config) - template = "Checking packages for os_type,'{os}' flavor '{flav}' and" \ - " ceph hash '{ver}'" + template = "Checking packages for os_type '{os}', " \ + "flavor '{flav}' and ceph hash '{ver}'" log.info( template.format( os=package.os_type, -- 2.39.5