From: Zack Cerza Date: Fri, 15 May 2015 19:13:18 +0000 (-0600) Subject: Use kernel_flavor when polling gitbuilders X-Git-Tag: 1.1.0~935^2~8 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7da76fa78f63a6ac1859a6c4b9097ba4acfa26e3;p=teuthology.git Use kernel_flavor when polling gitbuilders http://tracker.ceph.com/issues/11645 Signed-off-by: Zack Cerza --- diff --git a/teuthology/suite.py b/teuthology/suite.py index bf2cf5950..d19c5eca3 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -185,6 +185,7 @@ def create_initial_config(suite, suite_branch, ceph_branch, teuthology_branch, kernel_dict = dict(kernel=dict(kdb=True, sha1=kernel_hash)) else: kernel_dict = dict() + kernel_dict['kernel']['flavor'] = kernel_flavor # Get the ceph hash ceph_hash = get_hash('ceph', ceph_branch, kernel_flavor, machine_type, @@ -356,6 +357,7 @@ def get_hash(project='ceph', branch='master', flavor='basic', (arch, release, pkg_type) = get_distro_defaults(distro, machine_type) base_url = get_gitbuilder_url(project, release, pkg_type, arch, flavor) url = os.path.join(base_url, 'ref', branch, 'sha1') + log.debug("Gitbuilder URL: %s", url) resp = requests.get(url) if not resp.ok: return None @@ -516,6 +518,7 @@ def schedule_suite(job_config, parsed_yaml = yaml.load(raw_yaml) os_type = parsed_yaml.get('os_type') or job_config.os_type + kernel_flavor = job_config.kernel['flavor'] exclude_arch = parsed_yaml.get('exclude_arch') exclude_os_type = parsed_yaml.get('exclude_os_type') @@ -550,11 +553,14 @@ def schedule_suite(job_config, package_versions = get_package_versions( sha1, os_type, + kernel_flavor, package_versions ) - if not has_packages_for_distro(sha1, os_type, package_versions): - m = "Packages for os_type '{os}' and ceph hash '{ver}' not found" + if not has_packages_for_distro(sha1, os_type, kernel_flavor, + package_versions): + m = "Packages for os_type '{os}', flavor {flavor} and " + \ + "ceph hash '{ver}' not found" log.info(m.format(os=os_type, ver=sha1)) jobs_missing_packages.append(job) @@ -594,10 +600,10 @@ def schedule_suite(job_config, return count -def get_package_versions(sha1, os_type, package_versions=None): +def get_package_versions(sha1, os_type, kernel_flavor, package_versions=None): """ - Will retrieve the package versions for the given sha1 and os_type - from gitbuilder. + Will retrieve the package versions for the given sha1, os_type and + kernel_flavor from gitbuilder. Optionally, a package_versions dict can be provided from previous calls to this function to avoid calling gitbuilder for @@ -607,17 +613,24 @@ def get_package_versions(sha1, os_type, package_versions=None): { "sha1": { - "ubuntu": "version", - "rhel": "version", + "ubuntu": { + "basic": "version", + } + "rhel": { + "basic": "version", + } }, "another-sha1": { - "ubuntu": "version", + "ubuntu": { + "basic": "version", + } } } :param sha1: The sha1 hash of the ceph version. :param os_type: The distro we want to get packages for, given the ceph sha1. Ex. 'ubuntu', 'rhel', etc. + :param kernel_flavor: The kernel flavor :param package_versions: Use this optionally to use cached results of previous calls to gitbuilder. :returns: A dict of package versions. Will return versions @@ -629,21 +642,26 @@ def get_package_versions(sha1, os_type, package_versions=None): os_type = str(os_type) - package_versions_for_hash = package_versions.get(sha1, dict()) - if os_type not in package_versions_for_hash: + os_types = package_versions.get(sha1, dict()) + package_versions_for_flavor = os_types.get(os_type, dict()) + if kernel_flavor not in package_versions_for_flavor: package_version = package_version_for_hash( sha1, + kernel_flavor, distro=os_type ) - package_versions_for_hash[os_type] = package_version - package_versions[sha1] = package_versions_for_hash + package_versions_for_flavor[kernel_flavor] = package_version + os_types[os_type] = package_versions_for_flavor + package_versions[sha1] = os_types return package_versions -def has_packages_for_distro(sha1, os_type, package_versions=None): +def has_packages_for_distro(sha1, os_type, kernel_flavor, + package_versions=None): """ - Checks to see if gitbuilder has packages for the given sha1 and os_type. + Checks to see if gitbuilder has packages for the given sha1, os_type and + kernel_flavor. Optionally, a package_versions dict can be provided from previous calls to this function to avoid calling gitbuilder for @@ -653,28 +671,36 @@ def has_packages_for_distro(sha1, os_type, package_versions=None): { "sha1": { - "ubuntu": "version", - "rhel": "version", + "ubuntu": { + "basic": "version", + } + "rhel": { + "basic": "version", + } }, "another-sha1": { - "ubuntu": "version", + "ubuntu": { + "basic": "version", + } } } :param sha1: The sha1 hash of the ceph version. :param os_type: The distro we want to get packages for, given the ceph sha1. Ex. 'ubuntu', 'rhel', etc. + :param kernel_flavor: The kernel flavor :param package_versions: Use this optionally to use cached results of previous calls to gitbuilder. :returns: True, if packages are found. False otherwise. """ os_type = str(os_type) if package_versions is None: - package_versions = get_package_versions(sha1, os_type) + package_versions = get_package_versions(sha1, os_type, kernel_flavor) - package_versions_for_hash = package_versions.get(sha1, dict()) + package_versions_for_hash = package_versions.get(sha1, dict()).get( + os_type, dict()) # we want to return a boolean here, not the actual package versions - return bool(package_versions_for_hash.get(os_type, None)) + return bool(package_versions_for_hash.get(kernel_flavor, None)) def combine_path(left, right): diff --git a/teuthology/test/test_suite.py b/teuthology/test/test_suite.py index e50031c0b..4ec756f0e 100644 --- a/teuthology/test/test_suite.py +++ b/teuthology/test/test_suite.py @@ -143,7 +143,9 @@ class TestMissingPackages(object): def setup(self): package_versions = dict( sha1=dict( - ubuntu="1.0" + ubuntu=dict( + basic="1.0", + ) ) ) self.pv = package_versions @@ -152,6 +154,7 @@ class TestMissingPackages(object): assert self.pv == suite.get_package_versions( "sha1", "ubuntu", + "basic", package_versions=self.pv ) @@ -161,10 +164,11 @@ class TestMissingPackages(object): result = suite.get_package_versions( "sha1", "rhel", + "basic", package_versions=self.pv ) expected = deepcopy(self.pv) - expected['sha1'].update(dict(rhel="1.1")) + expected['sha1'].update(dict(rhel=dict(basic="1.1"))) assert result == expected @patch("teuthology.suite.package_version_for_hash") @@ -174,6 +178,7 @@ class TestMissingPackages(object): result = suite.get_package_versions( "sha1", "rhel", + "basic", package_versions=self.pv ) assert result == self.pv @@ -184,6 +189,7 @@ class TestMissingPackages(object): result = suite.get_package_versions( "sha1", "ubuntu", + "basic", ) expected = deepcopy(self.pv) assert result == expected @@ -192,6 +198,7 @@ class TestMissingPackages(object): result = suite.has_packages_for_distro( "sha1", "ubuntu", + "basic", package_versions=self.pv, ) assert result @@ -200,6 +207,7 @@ class TestMissingPackages(object): result = suite.has_packages_for_distro( "sha1", "rhel", + "basic", package_versions=self.pv, ) assert not result @@ -210,6 +218,7 @@ class TestMissingPackages(object): result = suite.has_packages_for_distro( "sha1", "rhel", + "basic", ) assert not result