From e6d12cade700d43df34a8dbc3a0b6b0b109b46b2 Mon Sep 17 00:00:00 2001 From: Andrew Schoen Date: Wed, 12 Aug 2015 11:48:02 -0500 Subject: [PATCH] task.install: use packing.GitbuilderProject to interface with gitbuilder This refactors the install and upgrade methods in the install task to use packing.GitbuilderProject for both the rpm and deb code paths. Signed-off-by: Andrew Schoen --- teuthology/packaging.py | 33 ++- teuthology/task/install.py | 294 ++++----------------------- teuthology/test/task/test_install.py | 141 ++----------- teuthology/test/test_packaging.py | 24 +++ 4 files changed, 108 insertions(+), 384 deletions(-) diff --git a/teuthology/packaging.py b/teuthology/packaging.py index 4723bd3706..04eaae1c5d 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -443,6 +443,8 @@ class GitbuilderProject(object): else: self._init_from_config() + self.dist_release = self._get_dist_release() + def _init_from_remote(self): """ Initializes the class from a teuthology.orchestra.remote.Remote object @@ -516,6 +518,16 @@ class GitbuilderProject(object): """ return self._get_uri_reference() + def _get_dist_release(self): + version = self._parse_version(self.os_version) + if self.os_type in ('centos', 'rhel'): + return "el{0}".format(version) + elif self.os_type == "fedora": + return "fc{0}".format(version) + else: + # debian and ubuntu just use the distro name + return self.os_type + def _parse_version(self, version): """ Parses a distro version string and returns a modified string @@ -610,12 +622,12 @@ class GitbuilderProject(object): """ tag = branch = sha1 = None if self.remote: - tag = _get_config_value_for_remote(self.ctx, self.job_config, - self.remote, 'tag') - branch = _get_config_value_for_remote(self.ctx, self.job_config, - self.remote, 'branch') - sha1 = _get_config_value_for_remote(self.ctx, self.job_config, - self.remote, 'sha1') + tag = _get_config_value_for_remote(self.ctx, self.remote, + self.job_config, 'tag') + branch = _get_config_value_for_remote(self.ctx, self.remote, + self.job_config, 'branch') + sha1 = _get_config_value_for_remote(self.ctx, self.remote, + self.job_config, 'sha1') else: sha1 = self.sha1 @@ -671,9 +683,10 @@ class GitbuilderProject(object): # repo file: v0.67-rc3.164.gd5aa3a9 - whereas in reality the RPM # version is 0.61.7 and the release is 37.g1243c97.el6 (centos6). # Point being, I have to mangle a little here. - if version[0] == 'v': - version = version[1:] - if '-' in version: - version = version.split('-')[0] + if self.pkg_type == "rpm": + if version[0] == 'v': + version = version[1:] + if '-' in version: + version = version.split('-')[0] log.info("Found version: {0}".format(version)) return version diff --git a/teuthology/task/install.py b/teuthology/task/install.py index 8c210b73c1..72c45cd276 100644 --- a/teuthology/task/install.py +++ b/teuthology/task/install.py @@ -5,13 +5,11 @@ import copy import logging import time import os -import requests import subprocess from teuthology.config import config as teuth_config from teuthology import misc as teuthology from teuthology import contextutil, packaging -from teuthology.exceptions import VersionNotFoundError from teuthology.parallel import parallel from ..orchestra import run @@ -64,178 +62,15 @@ PACKAGES['ceph']['rpm'] = [ 'rbd-fuse', ] -def _get_config_value_for_remote(ctx, remote, config, key): - """ - Look through config, and attempt to determine the "best" value to use for a - given key. For example, given: - - config = { - 'all': - {'branch': 'master'}, - 'branch': 'next' - } - _get_config_value_for_remote(ctx, remote, config, 'branch') - - would return 'master'. - - :param ctx: the argparse.Namespace object - :param remote: the teuthology.orchestra.remote.Remote object - :param config: the config dict - :param key: the name of the value to retrieve - """ - roles = ctx.cluster.remotes[remote] - if 'all' in config: - return config['all'].get(key) - elif roles: - for role in roles: - if role in config and key in config[role]: - return config[role].get(key) - return config.get(key) - - -def _get_uri(tag, branch, sha1): - """ - Set the uri -- common code used by both install and debian upgrade - """ - uri = None - if tag: - uri = 'ref/' + tag - elif branch: - uri = 'ref/' + branch - elif sha1: - uri = 'sha1/' + sha1 - else: - # FIXME: Should master be the default? - log.debug("defaulting to master branch") - uri = 'ref/master' - return uri - - -def _get_baseurlinfo_and_dist(ctx, remote, config): - """ - Through various commands executed on the remote, builds a dict which - will return the necessary information needed to build the url for - teuthology.config.baseurl_template. For example:: - - { - 'arch': 'x86_64', - 'dist': 'Centos7', - 'flavor': 'basic', - 'uri': 'ref/master', - 'dist_release': 'el7', - } - - :param ctx: the argparse.Namespace object - :param remote: the teuthology.orchestra.remote.Remote object - :param config: the config dict - :returns: dict -- the information you want. - """ - result = dict() - result['arch'] = remote.arch - - distro = remote.os.name - version = _get_gitbuilder_version(remote.os.version) - if distro in ('centos', 'rhel'): - distro = "centos" - dist_release = "el{0}".format(version) - elif distro == "fedora": - distro = "fc" - dist_release = "fc{0}".format(version) - else: - distro = remote.os.codename - dist_release = distro - version = "" - result['dist'] = "{distro}{version}".format( - distro=distro, - version=version, +def _get_gitbuilder_project(ctx, remote, config): + return packaging.GitbuilderProject( + config.get('project', 'ceph'), + config, + remote=remote, + ctx=ctx ) - # this is used when constructing the rpm name - result["dist_release"] = dist_release - - # branch/tag/sha1 flavor - result['flavor'] = config.get('flavor', 'basic') - log.info('config is %s', config) - tag = _get_config_value_for_remote(ctx, remote, config, 'tag') - branch = _get_config_value_for_remote(ctx, remote, config, 'branch') - sha1 = _get_config_value_for_remote(ctx, remote, config, 'sha1') - uri = _get_uri(tag, branch, sha1) - result['uri'] = uri - - return result - - -def _get_gitbuilder_version(version): - """ - Parses a distro version string and returns a modified string - that matches the format needed for the gitbuilder url. - - Minor version numbers are ignored. - """ - # return only the major version - return version.split(".")[0] - - -def _get_baseurl(ctx, remote, config): - """ - Figures out which package repo base URL to use. - - Example: - 'http://gitbuilder.ceph.com/ceph-deb-raring-x86_64-basic/ref/master' - :param ctx: the argparse.Namespace object - :param remote: the teuthology.orchestra.remote.Remote object - :param config: the config dict - :returns: str -- the URL - """ - template = teuth_config.baseurl_template - # get distro name and arch - baseparms = _get_baseurlinfo_and_dist(ctx, remote, config) - base_url = template.format( - host=teuth_config.gitbuilder_host, - proj=config.get('project', 'ceph'), - pkg_type=remote.system_type, - **baseparms - ) - return base_url - - -def _block_looking_for_package_version(remote, base_url, wait=False): - """ - Look for, and parse, a file called 'version' in base_url. - - :param remote: the teuthology.orchestra.remote.Remote object - :param wait: wait forever for the file to show up. (default False) - :returns: str -- the version e.g. '0.67-240-g67a95b9-1raring' - :raises: VersionNotFoundError - """ - url = "{0}/version".format(base_url) - log.info("Looking for package version: {0}".format(url)) - while True: - resp = requests.get(url) - if not resp.ok: - if wait: - log.info( - 'Package not there yet (got HTTP code %s), waiting...', - resp.status_code, - ) - time.sleep(15) - continue - raise VersionNotFoundError(base_url) - break - version = resp.text.strip() - # FIXME: 'version' as retreived from the repo is actually the RPM version - # PLUS *part* of the release. Example: - # Right now, ceph master is given the following version in the repo file: - # v0.67-rc3.164.gd5aa3a9 - whereas in reality the RPM version is 0.61.7 - # and the release is 37.g1243c97.el6 (for centos6). - # Point being, I have to mangle a little here. - if version[0] == 'v': - version = version[1:] - if '-' in version: - version = version.split('-')[0] - log.info("Found version: {0}".format(version)) - return version def _get_local_dir(config, remote): """ @@ -250,6 +85,7 @@ def _get_local_dir(config, remote): teuthology.sudo_write_file(remote, fname, open(fname).read(), '644') return ldir + def _update_deb_package_list_and_install(ctx, remote, debs, config): """ Runs ``apt-get update`` first, then runs ``apt-get install``, installing @@ -283,35 +119,19 @@ def _update_deb_package_list_and_install(ctx, remote, debs, config): stdout=StringIO(), ) - baseparms = _get_baseurlinfo_and_dist(ctx, remote, config) + gitbuilder = _get_gitbuilder_project(ctx, remote, config) log.info("Installing packages: {pkglist} on remote deb {arch}".format( - pkglist=", ".join(debs), arch=baseparms['arch']) + pkglist=", ".join(debs), arch=gitbuilder.arch) ) # get baseurl - base_url = _get_baseurl(ctx, remote, config) - log.info('Pulling from %s', base_url) + log.info('Pulling from %s', gitbuilder.base_url) - # get package version string - # FIXME this is a terrible hack. - while True: - resp = requests.get(base_url + '/version') - if not resp.ok: - if config.get('wait_for_package'): - log.info('Package not there yet, waiting...') - time.sleep(15) - continue - try: - resp.raise_for_status() - except Exception: - log.exception("Error fetching package version") - raise VersionNotFoundError("%s/version" % base_url) - version = resp.text.strip() - log.info('Package version is %s', version) - break + version = gitbuilder.version + log.info('Package version is %s', version) remote.run( args=[ - 'echo', 'deb', base_url, baseparms['dist'], 'main', + 'echo', 'deb', gitbuilder.base_url, gitbuilder.distro, 'main', run.Raw('|'), 'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format( proj=config.get('project', 'ceph')), @@ -411,12 +231,14 @@ def _update_rpm_package_list_and_install(ctx, remote, rpm, config): :param rpm: list of packages names to install :param config: the config dict """ - baseparms = _get_baseurlinfo_and_dist(ctx, remote, config) + gitbuilder = _get_gitbuilder_project(ctx, remote, config) + log.info('Pulling from %s', gitbuilder.base_url) + log.info('Package version is %s', gitbuilder.version) log.info("Installing packages: {pkglist} on remote rpm {arch}".format( - pkglist=", ".join(rpm), arch=baseparms['arch'])) - dist_release = baseparms['dist_release'] - project = config.get('project', 'ceph') - start_of_url = _get_baseurl(ctx, remote, config) + pkglist=", ".join(rpm), arch=gitbuilder.arch)) + dist_release = gitbuilder.dist_release + project = gitbuilder.project + start_of_url = gitbuilder.base_url proj_release = '{proj}-release-{release}.{dist_release}.noarch'.format( proj=project, release=RELEASE, dist_release=dist_release) rpm_name = "{rpm_nm}.rpm".format(rpm_nm=proj_release) @@ -429,7 +251,7 @@ def _update_rpm_package_list_and_install(ctx, remote, rpm, config): remote.run(args=['rm', '-f', rpm_name]) - uri = baseparms['uri'] + uri = gitbuilder.uri_reference _yum_fix_repo_priority(remote, project, uri) _yum_fix_repo_host(remote, project) _yum_set_check_obsoletes(remote) @@ -480,13 +302,9 @@ def verify_package_version(ctx, config, remote): if config.get("extras"): log.info("Skipping version verification...") return True - base_url = _get_baseurl(ctx, remote, config) - version = _block_looking_for_package_version( - remote, - base_url, - config.get('wait_for_package', False) - ) - pkg_to_check = config.get('project', 'ceph') + gitbuilder = _get_gitbuilder_project(ctx, remote, config) + version = gitbuilder.version + pkg_to_check = gitbuilder.project installed_ver = packaging.get_package_version(remote, pkg_to_check) if installed_ver and version in installed_ver: msg = "The correct {pkg} version {ver} is installed.".format( @@ -632,8 +450,8 @@ def _remove_rpm(ctx, config, remote, rpm): """ log.info("Removing packages: {pkglist} on rpm system.".format( pkglist=", ".join(rpm))) - baseparms = _get_baseurlinfo_and_dist(ctx, remote, config) - dist_release = baseparms['dist_release'] + gitbuilder = _get_gitbuilder_project(ctx, remote, config) + dist_release = gitbuilder.dist_release remote.run( args=[ 'for', 'd', 'in', @@ -858,47 +676,15 @@ def _upgrade_deb_packages(ctx, config, remote, debs): stdout=StringIO(), ) + gitbuilder = _get_gitbuilder_project(ctx, remote, config) # get distro name and arch - r = remote.run( - args=['lsb_release', '-sc'], - stdout=StringIO(), - ) - dist = r.stdout.getvalue().strip() - r = remote.run( - args=['arch'], - stdout=StringIO(), - ) - arch = r.stdout.getvalue().strip() - log.info("dist %s arch %s", dist, arch) - - # branch/tag/sha1 flavor - flavor = 'basic' - sha1 = config.get('sha1') - branch = config.get('branch') - tag = config.get('tag') - uri = _get_uri(tag, branch, sha1) - base_url = 'http://{host}/{proj}-deb-{dist}-{arch}-{flavor}/{uri}'.format( - host=teuth_config.gitbuilder_host, - proj=config.get('project', 'ceph'), - dist=dist, - arch=arch, - flavor=flavor, - uri=uri, - ) + dist = gitbuilder.distro + base_url = gitbuilder.base_url log.info('Pulling from %s', base_url) - # get package version string - while True: - resp = requests.get(base_url + '/version') - if not resp.ok: - if config.get('wait_for_package'): - log.info('Package not there yet, waiting...') - time.sleep(15) - continue - raise VersionNotFoundError("%s/version" % base_url) - version = resp.text.strip() - log.info('Package version is %s', version) - break + version = gitbuilder.version + log.info('Package version is %s', version) + remote.run( args=[ 'echo', 'deb', base_url, dist, 'main', @@ -1047,18 +833,18 @@ def _upgrade_rpm_packages(ctx, config, remote, pkgs): :param pkgs: the RPM packages to be installed :param branch: the branch of the project to be used """ - distinfo = _get_baseurlinfo_and_dist(ctx, remote, config) + gitbuilder = _get_gitbuilder_project(ctx, remote, config) log.info( "Host {host} is: {distro} {ver} {arch}".format( host=remote.shortname, - distro=distinfo['distro'], - ver=distinfo['relval'], - arch=distinfo['arch'],) + distro=gitbuilder.os_type, + ver=gitbuilder.os_version, + arch=gitbuilder.arch,) ) - base_url = _get_baseurl(ctx, remote, config) + base_url = gitbuilder.base_url log.info('Repo base URL: %s', base_url) - project = config.get('project', 'ceph') + project = gitbuilder.project # Remove the -release package before upgrading it args = ['sudo', 'rpm', '-ev', '%s-release' % project] @@ -1069,13 +855,13 @@ def _upgrade_rpm_packages(ctx, config, remote, pkgs): base=base_url, proj=project, release=RELEASE, - dist_release=distinfo['dist'], + dist_release=gitbuilder.dist_release, ) # Upgrade the -release package args = ['sudo', 'rpm', '-Uv', release_rpm] remote.run(args=args) - uri = _get_baseurlinfo_and_dist(ctx, remote, config)['uri'] + uri = gitbuilder.uri_reference _yum_fix_repo_priority(remote, project, uri) _yum_fix_repo_host(remote, project) _yum_set_check_obsoletes(remote) diff --git a/teuthology/test/task/test_install.py b/teuthology/test/task/test_install.py index 21bf6eed95..178d4acfc9 100644 --- a/teuthology/test/task/test_install.py +++ b/teuthology/test/task/test_install.py @@ -7,45 +7,25 @@ from teuthology.task import install class TestInstall(object): - @patch("teuthology.task.install.teuth_config") - @patch("teuthology.task.install._get_baseurlinfo_and_dist") - def test_get_baseurl(self, m_get_baseurlinfo_and_dist, m_config): - m_config.gitbuilder_host = 'FQDN' - config = {'project': 'CEPH'} - remote = Mock() - remote.system_type = 'rpm' - m_config.baseurl_template = ( - 'OVERRIDE: {host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}') - baseurlinfo_and_dist = { - 'dist': 'centos7', - 'arch': 'i386', - 'flavor': 'notcmalloc', - 'uri': 'ref/master', - } - m_get_baseurlinfo_and_dist.return_value = baseurlinfo_and_dist - expected = m_config.baseurl_template.format( - host=m_config.gitbuilder_host, - proj=config['project'], - pkg_type=remote.system_type, - **baseurlinfo_and_dist) - actual = install._get_baseurl(Mock(), remote, config) - assert expected == actual - - @patch("teuthology.task.install._get_baseurl") - @patch("teuthology.task.install._block_looking_for_package_version") + @patch("teuthology.task.install._get_gitbuilder_project") @patch("teuthology.task.install.packaging.get_package_version") - def test_verify_ceph_version_success(self, m_get_package_version, m_block, - m_get_baseurl): - m_block.return_value = "0.89.0" + def test_verify_ceph_version_success(self, m_get_package_version, + m_gitbuilder_project): + gb = Mock() + gb.version = "0.89.0" + gb.project = "ceph" + m_gitbuilder_project.return_value = gb m_get_package_version.return_value = "0.89.0" install.verify_package_version(Mock(), Mock(), Mock()) - @patch("teuthology.task.install._get_baseurl") - @patch("teuthology.task.install._block_looking_for_package_version") + @patch("teuthology.task.install._get_gitbuilder_project") @patch("teuthology.task.install.packaging.get_package_version") - def test_verify_ceph_version_failed(self, m_get_package_version, m_block, - m_get_baseurl): - m_block.return_value = "0.89.0" + def test_verify_ceph_version_failed(self, m_get_package_version, + m_gitbuilder_project): + gb = Mock() + gb.version = "0.89.0" + gb.project = "ceph" + m_gitbuilder_project.return_value = gb m_get_package_version.return_value = "0.89.1" config = Mock() # when it looks for config.get('extras') it won't find it @@ -53,12 +33,14 @@ class TestInstall(object): with pytest.raises(RuntimeError): install.verify_package_version(Mock(), config, Mock()) - @patch("teuthology.task.install._get_baseurl") - @patch("teuthology.task.install._block_looking_for_package_version") + @patch("teuthology.task.install._get_gitbuilder_project") @patch("teuthology.task.install.packaging.get_package_version") - def test_skip_when_using_ceph_deploy(self, m_get_package_version, m_block, - m_get_baseurl): - m_block.return_value = "0.89.0" + def test_skip_when_using_ceph_deploy(self, m_get_package_version, + m_gitbuilder_project): + gb = Mock() + gb.version = "0.89.0" + gb.project = "ceph" + m_gitbuilder_project.return_value = gb # ceph isn't installed because ceph-deploy would install it m_get_package_version.return_value = None config = Mock() @@ -80,84 +62,3 @@ class TestInstall(object): valgrind=True ) assert install.get_flavor(config) == 'notcmalloc' - - @patch("teuthology.task.install._get_config_value_for_remote") - def test_get_baseurlinfo_and_dist_rhel(self, m_get_config_value_for_remote): - remote = Mock() - remote.os.name = "rhel" - remote.os.version = "7.0" - remote.arch = "x86_64" - m_get_config_value_for_remote.return_value = "tag" - result = install._get_baseurlinfo_and_dist(Mock(), remote, dict()) - expected = dict( - dist="centos7", - arch="x86_64", - flavor="basic", - uri="ref/tag", - dist_release="el7" - ) - assert result == expected - - @patch("teuthology.task.install._get_config_value_for_remote") - def test_get_baseurlinfo_and_dist_minor_version_allowed(self, m_get_config_value_for_remote): - remote = Mock() - remote.os.name = "centos" - remote.os.version = "6.5" - remote.arch = "x86_64" - m_get_config_value_for_remote.return_value = "tag" - result = install._get_baseurlinfo_and_dist(Mock(), remote, dict()) - expected = dict( - dist="centos6", - arch="x86_64", - flavor="basic", - uri="ref/tag", - dist_release="el6", - ) - assert result == expected - - @patch("teuthology.task.install._get_config_value_for_remote") - def test_get_baseurlinfo_and_dist_fedora(self, m_get_config_value_for_remote): - remote = Mock() - remote.os.name = "fedora" - remote.os.version = "20" - remote.arch = "x86_64" - m_get_config_value_for_remote.return_value = "tag" - result = install._get_baseurlinfo_and_dist(Mock(), remote, dict()) - expected = dict( - dist="fc20", - arch="x86_64", - flavor="basic", - uri="ref/tag", - dist_release="fc20", - ) - assert result == expected - - @patch("teuthology.task.install._get_config_value_for_remote") - def test_get_baseurlinfo_and_dist_ubuntu(self, m_get_config_value_for_remote): - remote = Mock() - remote.os.name = "ubuntu" - remote.os.version = "14.04" - remote.os.codename = "trusty" - remote.arch = "x86_64" - m_get_config_value_for_remote.return_value = "tag" - result = install._get_baseurlinfo_and_dist(Mock(), remote, dict()) - expected = dict( - dist="trusty", - arch="x86_64", - flavor="basic", - uri="ref/tag", - dist_release="trusty", - ) - assert result == expected - - def test_get_uri_tag(self): - assert "ref/thetag" == install._get_uri("thetag", None, None) - - def test_get_uri_branch(self): - assert "ref/thebranch" == install._get_uri(None, "thebranch", None) - - def test_get_uri_sha1(self): - assert "sha1/thesha1" == install._get_uri(None, None, "thesha1") - - def test_get_uri_default(self): - assert "ref/master" == install._get_uri(None, None, None) diff --git a/teuthology/test/test_packaging.py b/teuthology/test/test_packaging.py index 7559b817d0..2fc9e4e418 100644 --- a/teuthology/test/test_packaging.py +++ b/teuthology/test/test_packaging.py @@ -407,3 +407,27 @@ class TestGitbuilderProject(object): ) gp = packaging.GitbuilderProject("ceph", config) assert gp.distro == expected + + GITBUILDER_DIST_RELEASE_MATRIX = [ + ('rhel', '7.0', None, 'el7'), + ('centos', '6.5', None, 'el6'), + ('centos', '7.0', None, 'el7'), + ('centos', '7.1', None, 'el7'), + ('fedora', '20', None, 'fc20'), + ('debian', '7.0', None, 'debian'), + ('debian', '7', None, 'debian'), + ('debian', '7.1', None, 'debian'), + ('ubuntu', '12.04', None, 'ubuntu'), + ('ubuntu', '14.04', None, 'ubuntu'), + ] + + @pytest.mark.parametrize( + "distro, version, codename, expected", + GITBUILDER_DIST_RELEASE_MATRIX + ) + def test_get_dist_release(self, distro, version, codename, expected): + rem = self._get_remote(distro=distro, version=version, + codename=codename) + ctx = dict(foo="bar") + gp = packaging.GitbuilderProject("ceph", {}, ctx=ctx, remote=rem) + assert gp.dist_release == expected -- 2.39.5