From c80939ff7d70afb9de746cd87e6833f0ce1e1b8f Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 9 Nov 2021 11:49:09 -0800 Subject: [PATCH] teuthology/packaging.py: fix build_complete: search for requested arch The workaround from https://github.com/ceph/teuthology/pull/1649 was necessary because my original algorithm was faulty: when searching through all the builds for a ref/sha1, one must match the arch requested by the call to build_complete (in the Builder object); that arch's presence in the shaman api/search result is not enough of a match, as it can contain multiple arches in multiple states of build success. Only a failure *on the requested arch* should be considered a "requested build not complete". (note: this will still currently fail a request for a build whose repo is complete but container build failed, as "build complete" currently conflates those two statuses. Teuthology does not contain the information whether a build is being requested for packages, containers, or both.) Also add testing for build_complete(). Fixes: https://tracker.ceph.com/issues/53205 Signed-off-by: Dan Mick --- teuthology/packaging.py | 10 ++- teuthology/test/test_packaging.py | 116 ++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/teuthology/packaging.py b/teuthology/packaging.py index 7842339a..9d30a2bf 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -977,6 +977,9 @@ class ShamanProject(GitbuilderProject): except VersionNotFoundError: return False + # self._result has status, project, flavor, distros, arch, and sha1 + # restrictions, so the only reason for multiples should be "multiple + # builds of the same sha1 etc."; the first entry is the newest search_result = self._result.json()[0] # now look for the build complete status @@ -990,12 +993,17 @@ class ShamanProject(GitbuilderProject): resp.raise_for_status() except requests.HttpError: return False + log.debug(f'looking for {self.distro} {self.arch} {self.flavor}') for build in resp.json(): + log.debug(f'build: {build["distro"]}/{build["distro_version"]} {build["distro_arch"]} {build["flavor"]}') if ( + # we must compare build arch to self.arch, since shaman's + # results can have multiple arches but we're searching + # for precisely one here build['distro'] == search_result['distro'] and build['distro_version'] == search_result['distro_version'] and build['flavor'] == search_result['flavor'] and - build['distro_arch'] == 'x86_64' + build['distro_arch'] == self.arch ): return build['status'] == 'completed' return False diff --git a/teuthology/test/test_packaging.py b/teuthology/test/test_packaging.py index baeba956..98db2e38 100644 --- a/teuthology/test/test_packaging.py +++ b/teuthology/test/test_packaging.py @@ -650,6 +650,122 @@ class TestShamanProject(TestBuilderProject): super(TestShamanProject, self)\ .test_get_package_sha1_fetched_not_found() + SHAMAN_SEARCH_RESPONSE = [ + { + "status": "ready", + "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323", + "extra": { + "build_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/", + "root_build_cause": "SCMTRIGGER", + "version": "17.0.0-8856-g534fc6d9", + "node_name": "172.21.2.7+braggi07", + "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic", + "package_manager_version": "17.0.0-8856.g534fc6d9" + }, + "url": "https://3.chacra.ceph.com/r/ceph/master/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/", + "modified": "2021-11-06 21:40:40.669823", + "distro_version": "8", + "project": "ceph", + "flavor": "default", + "ref": "master", + "chacra_url": "https://3.chacra.ceph.com/repos/ceph/master/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/", + "archs": [ + "x86_64", + "arm64", + "source" + ], + "distro": "centos" + } + ] + + SHAMAN_BUILDS_RESPONSE = [ + { + "status": "completed", + "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323", + "distro_arch": "arm64", + "started": "2021-11-06 20:20:15.121203", + "completed": "2021-11-06 22:36:27.115950", + "extra": { + "node_name": "172.21.4.66+confusa04", + "version": "17.0.0-8856-g534fc6d9", + "build_user": "", + "root_build_cause": "SCMTRIGGER", + + "job_name": "ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic" + }, + "modified": "2021-11-06 22:36:27.118043", + "distro_version": "8", + "project": "ceph", + "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/", + "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull", + "flavor": "default", + "ref": "master", + "distro": "centos" + }, + { + "status": "completed", + "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323", + "distro_arch": "x86_64", + "started": "2021-11-06 20:20:06.740692", + "completed": "2021-11-06 21:43:51.711970", + "extra": { + "node_name": "172.21.2.7+braggi07", + "version": "17.0.0-8856-g534fc6d9", + "build_user": "", + "root_build_cause": "SCMTRIGGER", + "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic" + }, + "modified": "2021-11-06 21:43:51.713487", + "distro_version": "8", + "project": "ceph", + "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/", + "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull", + "flavor": "default", + "ref": "master", + "distro": "centos" + } + ] + + def test_build_complete_success(self): + config = dict( + os_type="centos", + os_version="8", + branch='master', + arch='x86_64', + flavor='default', + ) + builder = self.klass("ceph", config) + + search_resp = Mock() + search_resp.ok = True + search_resp.json.return_value = self.SHAMAN_SEARCH_RESPONSE + self.m_get.return_value = search_resp + # cause builder to call requests.get and cache search_resp + builder.assert_result() + + build_resp = Mock() + build_resp.ok = True + self.m_get.return_value = build_resp + + # both archs completed, so x86_64 build is complete + builds = build_resp.json.return_value = self.SHAMAN_BUILDS_RESPONSE + assert builder.build_complete + + # mark the arm64 build failed, x86_64 should still be complete + builds[0]['status'] = "failed" + build_resp.json.return_value = builds + assert builder.build_complete + + # mark the x86_64 build failed, should show incomplete + builds[1]['status'] = "failed" + build_resp.json.return_value = builds + assert not builder.build_complete + + # mark the arm64 build complete again, x86_64 still incomplete + builds[0]['status'] = "completed" + build_resp.json.return_value = builds + assert not builder.build_complete + DISTRO_MATRIX = [ ('rhel', '7.0', None, 'centos/7'), ('centos', '6.5', None, 'centos/6'), -- 2.47.3