teuthology/packaging.py: fix build_complete: search for requested arch

The workaround from https://github.com/ceph/teuthology/pull/1649 was
necessary because my original algorithm was faulty: when searching
through all the builds for a ref/sha1, one must match the arch
requested by the call to build_complete (in the Builder object);
that arch's presence in the shaman api/search result is not enough
of a match, as it can contain multiple arches in multiple states
of build success.  Only a failure *on the requested arch* should be
considered a "requested build not complete".

(note: this will still currently fail a request for a build whose
repo is complete but container build failed, as "build complete"
currently conflates those two statuses.  Teuthology does not
contain the information whether a build is being requested for
packages, containers, or both.)

Also add testing for build_complete().

Fixes: https://tracker.ceph.com/issues/53205
Signed-off-by: Dan Mick <dmick@redhat.com>

diff --git a/teuthology/packaging.py b/teuthology/packaging.py
index 7842339a0097c0fe29932e20cc2c7d312d7582b0..9d30a2bfedcdedfd8f5076482ee3eff275a890e4 100644 (file)
--- a/teuthology/packaging.py
+++ b/teuthology/packaging.py
@@ -977,6 +977,9 @@ class ShamanProject(GitbuilderProject):
         except VersionNotFoundError:
             return False
 
+        # self._result has status, project, flavor, distros, arch, and sha1
+        # restrictions, so the only reason for multiples should be "multiple
+        # builds of the same sha1 etc."; the first entry is the newest
         search_result = self._result.json()[0]
 
         # now look for the build complete status
@@ -990,12 +993,17 @@ class ShamanProject(GitbuilderProject):
             resp.raise_for_status()
         except requests.HttpError:
             return False
+        log.debug(f'looking for {self.distro} {self.arch} {self.flavor}')
         for build in resp.json():
+            log.debug(f'build: {build["distro"]}/{build["distro_version"]} {build["distro_arch"]} {build["flavor"]}')
             if (
+                # we must compare build arch to self.arch, since shaman's
+                # results can have multiple arches but we're searching
+                # for precisely one here
                 build['distro'] == search_result['distro'] and
                 build['distro_version'] == search_result['distro_version'] and
                 build['flavor'] == search_result['flavor'] and
-                build['distro_arch'] == 'x86_64'
+                build['distro_arch'] == self.arch
                ):
                 return build['status'] == 'completed'
         return False

diff --git a/teuthology/test/test_packaging.py b/teuthology/test/test_packaging.py
index baeba956cc5d48ffa3ad8fd910e0938c29152029..98db2e380c8cb5c8739f283ca541dfcc530cd46a 100644 (file)
--- a/teuthology/test/test_packaging.py
+++ b/teuthology/test/test_packaging.py
@@ -650,6 +650,122 @@ class TestShamanProject(TestBuilderProject):
         super(TestShamanProject, self)\
             .test_get_package_sha1_fetched_not_found()
 
+    SHAMAN_SEARCH_RESPONSE = [
+        {
+            "status": "ready",
+            "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
+            "extra": {
+                "build_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
+                "root_build_cause": "SCMTRIGGER",
+                "version": "17.0.0-8856-g534fc6d9",
+                "node_name": "172.21.2.7+braggi07",
+                "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic",
+                "package_manager_version": "17.0.0-8856.g534fc6d9"
+            },
+            "url": "https://3.chacra.ceph.com/r/ceph/master/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/",
+            "modified": "2021-11-06 21:40:40.669823",
+            "distro_version": "8",
+            "project": "ceph",
+            "flavor": "default",
+            "ref": "master",
+            "chacra_url": "https://3.chacra.ceph.com/repos/ceph/master/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/",
+            "archs": [
+                "x86_64",
+                "arm64",
+                "source"
+            ],
+            "distro": "centos"
+          }
+    ]
+
+    SHAMAN_BUILDS_RESPONSE = [
+        {
+            "status": "completed",
+            "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
+            "distro_arch": "arm64",
+            "started": "2021-11-06 20:20:15.121203",
+            "completed": "2021-11-06 22:36:27.115950",
+            "extra": {
+                "node_name": "172.21.4.66+confusa04",
+                "version": "17.0.0-8856-g534fc6d9",
+                "build_user": "",
+                "root_build_cause": "SCMTRIGGER",
+
+                "job_name": "ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic"
+            },
+            "modified": "2021-11-06 22:36:27.118043",
+            "distro_version": "8",
+            "project": "ceph",
+            "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
+            "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull",
+            "flavor": "default",
+            "ref": "master",
+            "distro": "centos"
+        },
+        {
+            "status": "completed",
+            "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
+            "distro_arch": "x86_64",
+            "started": "2021-11-06 20:20:06.740692",
+            "completed": "2021-11-06 21:43:51.711970",
+            "extra": {
+                "node_name": "172.21.2.7+braggi07",
+                "version": "17.0.0-8856-g534fc6d9",
+                "build_user": "",
+                "root_build_cause": "SCMTRIGGER",
+                "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic"
+            },
+            "modified": "2021-11-06 21:43:51.713487",
+            "distro_version": "8",
+            "project": "ceph",
+            "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
+            "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull",
+            "flavor": "default",
+            "ref": "master",
+            "distro": "centos"
+        }
+    ]
+
+    def test_build_complete_success(self):
+        config = dict(
+            os_type="centos",
+            os_version="8",
+            branch='master',
+            arch='x86_64',
+            flavor='default',
+        )
+        builder = self.klass("ceph", config)
+
+        search_resp = Mock()
+        search_resp.ok = True
+        search_resp.json.return_value = self.SHAMAN_SEARCH_RESPONSE
+        self.m_get.return_value = search_resp
+        # cause builder to call requests.get and cache search_resp
+        builder.assert_result()
+
+        build_resp = Mock()
+        build_resp.ok = True
+        self.m_get.return_value = build_resp
+
+        # both archs completed, so x86_64 build is complete
+        builds = build_resp.json.return_value = self.SHAMAN_BUILDS_RESPONSE
+        assert builder.build_complete
+
+        # mark the arm64 build failed, x86_64 should still be complete
+        builds[0]['status'] = "failed"
+        build_resp.json.return_value = builds
+        assert builder.build_complete
+
+        # mark the x86_64 build failed, should show incomplete
+        builds[1]['status'] = "failed"
+        build_resp.json.return_value = builds
+        assert not builder.build_complete
+
+        # mark the arm64 build complete again, x86_64 still incomplete
+        builds[0]['status'] = "completed"
+        build_resp.json.return_value = builds
+        assert not builder.build_complete
+
     DISTRO_MATRIX = [
         ('rhel', '7.0', None, 'centos/7'),
         ('centos', '6.5', None, 'centos/6'),