import logging
import ast
import re
+import requests
from cStringIO import StringIO
'httpd': {'deb': 'apache2', 'rpm': 'httpd'}
}
+DISTRO_CODENAME_MAP = {
+ "ubuntu": {
+ "14.04": "trusty",
+ "12.04": "precise",
+ "15.04": "vivid",
+ },
+ "debian": {
+ "7": "wheezy",
+ },
+}
+
+DEFAULT_OS_VERSION = dict(
+ ubuntu="14.04",
+ fedora="20",
+ centos="7.0",
+ opensuse="12.2",
+ sles="11-sp2",
+ rhel="7.0",
+ debian='7.0'
+)
+
def get_package_name(pkg, rem):
"""
)
return installed_ver
+
+
+def _get_config_value_for_remote(ctx, remote, config, key):
+ """
+ Look through config, and attempt to determine the "best" value to use
+ for a given key. For example, given:
+
+ config = {
+ 'all':
+ {'branch': 'master'},
+ 'branch': 'next'
+ }
+ _get_config_value_for_remote(ctx, remote, config, 'branch')
+
+ would return 'master'.
+
+ :param ctx: the argparse.Namespace object
+ :param remote: the teuthology.orchestra.remote.Remote object
+ :param config: the config dict
+ :param key: the name of the value to retrieve
+ """
+ roles = ctx.cluster.remotes[remote]
+ if 'all' in config:
+ return config['all'].get(key)
+ elif roles:
+ for role in roles:
+ if role in config and key in config[role]:
+ return config[role].get(key)
+ return config.get(key)
+
+
+class GitbuilderProject(object):
+ """
+ Represents a project that is built by gitbuilder.
+ """
+
+ def __init__(self, project, job_config, ctx=None, remote=None):
+ self.project = project
+ self.job_config = job_config
+ self.ctx = ctx
+ self.remote = remote
+ # avoiding circular imports
+ from teuthology.suite import get_install_task_flavor
+ self.flavor = get_install_task_flavor(self.job_config)
+ self.sha1 = self.job_config.get("sha1")
+
+ if remote and ctx:
+ self._init_from_remote()
+ else:
+ self._init_from_config()
+
+ def _init_from_remote(self):
+ """
+ Initializes the class from a teuthology.orchestra.remote.Remote object
+ """
+ self.arch = self.remote.arch
+ self.os_type = self.remote.os.name
+ self.os_version = self.remote.os.version
+ self.pkg_type = self.remote.system_type
+ self.distro = self._get_distro(
+ distro=self.remote.os.name,
+ version=self.remote.os.version,
+ codename=self.remote.os.codename,
+ )
+
+ def _init_from_config(self):
+ """
+ Initializes the class from a teuthology job config
+ """
+ # a bad assumption, but correct for most situations I believe
+ self.arch = "x86_64"
+ self.os_type = self.job_config.get("os_type")
+ self.os_version = self._get_version()
+ self.distro = self._get_distro(
+ distro=self.os_type,
+ version=self.os_version,
+ )
+ self.pkg_type = "deb" if self.os_type.lower() in (
+ "ubuntu",
+ "debian",
+ ) else "rpm"
+
+ @property
+ def version(self):
+ if not hasattr(self, '_version'):
+ self._version = self._get_package_version()
+ return self._version
+
+ @property
+ def base_url(self):
+ return self._get_base_url()
+
+ @property
+ def uri(self):
+ return self._get_uri()
+
+ def _parse_version(self, version):
+ """
+ Parses a distro version string and returns a modified string
+ that matches the format needed for the gitbuilder url.
+
+ Minor version numbers are ignored if they end in a zero. If they do
+ not end in a zero the minor version number is included with an
+ underscore as the separator instead of a period.
+ """
+ version_tokens = version.split(".")
+ include_minor_version = (
+ len(version_tokens) > 1 and
+ version_tokens[1] != "0"
+ )
+ if include_minor_version:
+ return "_".join(version_tokens)
+
+ # return only the major version
+ return version_tokens[0]
+
+ def _get_distro(self, distro=None, version=None, codename=None):
+ """
+ Given a distro and a version, returned the combined string
+ to use in a gitbuilder url.
+
+ :param distro: The distro as a string
+ :param version: The version as a string
+ :param codename: The codename for the distro.
+ Used for deb based distros.
+ """
+ if distro in ('centos', 'rhel'):
+ distro = "centos"
+ elif distro == "fedora":
+ distro = "fc"
+ else:
+ # deb based systems use codename instead of a distro/version combo
+ if not codename:
+ # lookup codename based on distro string
+ codename = self._get_codename(distro, version)
+ if not codename:
+ msg = "No codename found for: {distro} {version}".format(
+ distro=distro,
+ version=version,
+ )
+ log.exception(msg)
+ raise RuntimeError()
+ return codename
+
+ return "{distro}{version}".format(
+ distro=distro,
+ version=self._parse_version(version),
+ )
+
+ def _get_codename(self, distro, version):
+ major_version = version.split(".")[0]
+ distro_codes = DISTRO_CODENAME_MAP.get(distro)
+ if not distro_codes:
+ return None
+ codename = distro_codes.get(version)
+ if not codename:
+ codename = distro_codes.get(major_version)
+
+ return codename
+
+ def _get_version(self):
+ version = self.job_config.get("os_version")
+ if not version:
+ version = DEFAULT_OS_VERSION.get(self.os_type)
+
+ return version
+
+ def _get_uri(self):
+ """
+ Set the uri -- common code used by both install and debian upgrade
+ """
+ tag = branch = sha1 = None
+ if self.remote:
+ tag = _get_config_value_for_remote(self.ctx, self.job_config,
+ self.remote, 'tag')
+ branch = _get_config_value_for_remote(self.ctx, self.job_config,
+ self.remote, 'branch')
+ sha1 = _get_config_value_for_remote(self.ctx, self.job_config,
+ self.remote, 'sha1')
+ else:
+ sha1 = self.sha1
+
+ if tag:
+ uri = 'ref/' + tag
+ elif branch:
+ uri = 'ref/' + branch
+ elif sha1:
+ uri = 'sha1/' + sha1
+ else:
+ # FIXME: Should master be the default?
+ log.debug("defaulting to master branch")
+ uri = 'ref/master'
+ return uri
+
+ def _get_base_url(self):
+ """
+ Figures out which package repo base URL to use.
+ """
+ template = config.baseurl_template
+ # get distro name and arch
+ base_url = template.format(
+ host=config.gitbuilder_host,
+ proj=self.project,
+ pkg_type=self.pkg_type,
+ arch=self.arch,
+ dist=self.distro,
+ flavor=self.flavor,
+ uri=self.uri,
+ )
+ return base_url
+
+ def _get_package_version(self):
+ """
+ Look for, and parse, a file called 'version' in base_url.
+ """
+ url = "{0}/version".format(self.base_url)
+ log.info("Looking for package version: {0}".format(url))
+ resp = requests.get(url)
+ version = None
+ if not resp.ok:
+ log.info(
+ 'Package not there yet (got HTTP code %s), waiting...',
+ resp.status_code,
+ )
+ else:
+ version = resp.text.strip()
+ # TODO: move this parsing into a different function for
+ # easier testing
+ # FIXME: 'version' as retreived from the repo is actually the
+ # RPM version PLUS *part* of the release. Example:
+ # Right now, ceph master is given the following version in the
+ # repo file: v0.67-rc3.164.gd5aa3a9 - whereas in reality the RPM
+ # version is 0.61.7 and the release is 37.g1243c97.el6 (centos6).
+ # Point being, I have to mangle a little here.
+ if version[0] == 'v':
+ version = version[1:]
+ if '-' in version:
+ version = version.split('-')[0]
+ log.info("Found version: {0}".format(version))
+ return version
@pytest.mark.parametrize("input, expected", KOJI_TASK_RPMS_MATRIX)
def test_get_koji_task_result_package_name(self, input, expected):
assert packaging._get_koji_task_result_package_name(input) == expected
+
+
+class TestGitbuilderProject(object):
+
+ def _get_remote(self, arch="x86_64", system_type="deb", distro="ubuntu",
+ codename="trusty", version="14.04"):
+ rem = Mock()
+ rem.system_type = system_type
+ rem.os.name = distro
+ rem.os.codename = codename
+ rem.os.version = version
+ rem.arch = arch
+
+ return rem
+
+ @patch("teuthology.packaging.config")
+ @patch("teuthology.packaging._get_config_value_for_remote")
+ def test_init_from_remote_base_url(self, m_get_config_value, m_config):
+ m_config.baseurl_template = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}'
+ m_config.gitbuilder_host = "gitbuilder.ceph.com"
+ m_get_config_value.return_value = None
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ gp = packaging.GitbuilderProject("ceph", {}, ctx=ctx, remote=rem)
+ result = gp.base_url
+ expected = "http://gitbuilder.ceph.com/ceph-deb-trusty-x86_64-basic/ref/master"
+ assert result == expected
+
+ @patch("teuthology.packaging.config")
+ def test_init_from_config_base_url(self, m_config):
+ m_config.baseurl_template = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}'
+ m_config.gitbuilder_host = "gitbuilder.ceph.com"
+ config = dict(
+ os_type="ubuntu",
+ os_version="14.04",
+ sha1="sha1",
+ )
+ gp = packaging.GitbuilderProject("ceph", config)
+ result = gp.base_url
+ expected = "http://gitbuilder.ceph.com/ceph-deb-trusty-x86_64-basic/sha1/sha1"
+ assert result == expected
+
+ @patch("teuthology.packaging.config")
+ @patch("teuthology.packaging._get_config_value_for_remote")
+ @patch("requests.get")
+ def test_get_package_version_found(self, m_get, m_get_config_value,
+ m_config):
+ m_config.baseurl_template = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}'
+ m_config.gitbuilder_host = "gitbuilder.ceph.com"
+ m_get_config_value.return_value = None
+ resp = Mock()
+ resp.ok = True
+ resp.text = "0.90.0"
+ m_get.return_value = resp
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ gp = packaging.GitbuilderProject("ceph", {}, ctx=ctx, remote=rem)
+ assert gp.version == "0.90.0"
+
+ @patch("teuthology.packaging.config")
+ @patch("teuthology.packaging._get_config_value_for_remote")
+ @patch("requests.get")
+ def test_get_package_version_not_found(self, m_get, m_get_config_value,
+ m_config):
+ m_config.baseurl_template = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}'
+ m_config.gitbuilder_host = "gitbuilder.ceph.com"
+ m_get_config_value.return_value = None
+ resp = Mock()
+ resp.ok = False
+ m_get.return_value = resp
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ gp = packaging.GitbuilderProject("ceph", {}, ctx=ctx, remote=rem)
+ assert not gp.version
+
+ GITBUILDER_DISTRO_MATRIX = [
+ ('rhel', '7.0', None, 'centos7'),
+ ('centos', '6.5', None, 'centos6_5'),
+ ('fedora', '20', None, 'fc20'),
+ ('ubuntu', '14.04', 'trusty', 'trusty'),
+ ('ubuntu', '14.04', None, 'trusty'),
+ ('debian', '7.0', None, 'wheezy'),
+ ('debian', '7', None, 'wheezy'),
+ ('ubuntu', '12.04', None, 'precise'),
+ ('ubuntu', '14.04', None, 'trusty'),
+ ]
+
+ @pytest.mark.parametrize(
+ "distro, version, codename, expected",
+ GITBUILDER_DISTRO_MATRIX
+ )
+ def test_get_distro_remote(self, distro, version, codename, expected):
+ rem = self._get_remote(distro=distro, version=version,
+ codename=codename)
+ ctx = dict(foo="bar")
+ gp = packaging.GitbuilderProject("ceph", {}, ctx=ctx, remote=rem)
+ assert gp.distro == expected
+
+ @pytest.mark.parametrize(
+ "distro, version, codename, expected",
+ GITBUILDER_DISTRO_MATRIX + [
+ ('rhel', None, None, 'centos7'),
+ ('centos', None, None, 'centos7'),
+ ('fedora', None, None, 'fc20'),
+ ('ubuntu', None, None, 'trusty'),
+ ('debian', None, None, 'wheezy'),
+ ]
+ )
+ def test_get_distro_config(self, distro, version, codename, expected):
+ config = dict(
+ os_type=distro,
+ os_version=version
+ )
+ gp = packaging.GitbuilderProject("ceph", config)
+ assert gp.distro == expected
projects / teuthology.git / commitdiff