As opposed to checking for a specific completed package build, and inferring the
status of the container based on the result. This check will only affect jobs
which use the cephadm task.
Signed-off-by: Zack Cerza <zack@cerza.org>
from teuthology.suite.merge import config_merge
from teuthology.suite.build_matrix import build_matrix
from teuthology.suite.placeholder import substitute_placeholders, dict_templ
+from teuthology.util.containers import container_image_for_hash
from teuthology.util.time import parse_offset, parse_timestamp, TIMESTAMP_FMT
log = logging.getLogger(__name__)
# no point in continuing the search
if newest:
return jobs_missing_packages, []
+ job_tasks = set()
+ for task_dict in parsed_yaml.get('tasks', []):
+ for key in task_dict.keys():
+ job_tasks.add(key)
+ if any(['cephadm' in name for name in job_tasks]):
+ if not container_image_for_hash(sha1):
+ jobs_missing_packages.append(job)
+ if newest:
+ return jobs_missing_packages, []
jobs_to_schedule.append(job)
return jobs_missing_packages, jobs_to_schedule
log = logging.getLogger(__name__)
-CONTAINER_DISTRO = 'centos/9' # the one to check for build_complete
-CONTAINER_FLAVOR = 'default'
-
def fetch_repos(branch, test_name, dry_run, commit=None):
"""
),
)
- if (bp.distro == CONTAINER_DISTRO and bp.flavor == CONTAINER_FLAVOR and
- not bp.build_complete):
- log.info("Container build incomplete")
- return None
-
try:
return bp.version
except VersionNotFoundError:
return None
-
def get_arch(machine_type):
"""
Based on a given machine_type, return its architecture by querying the lock
--- /dev/null
+import functools
+import logging
+import re
+import requests
+
+log = logging.getLogger(__name__)
+
+# Our container images use a certain base image and flavor by default; those
+# values are reflected below. If different values are used, they are appended
+# to the image name.
+DEFAULT_CONTAINER_BASE = 'centos:9'
+DEFAULT_CONTAINER_FLAVOR = 'default'
+DEFAULT_CONTAINER_IMAGE='quay.ceph.io/ceph-ci/ceph:{sha1}'
+CONTAINER_REGEXP = re.compile(
+ r"((?P<domain>[a-zA-Z0-9._-]+)/)?((?P<org>[a-zA-Z0-9_-]+)/)?((?P<image>[a-zA-Z0-9_-]+))?(:(?P<tag>[a-zA-Z0-9._-]+))?"
+)
+
+
+def resolve_container_image(image: str):
+ """
+ Given an image locator that is potentially incomplete, construct a qualified version.
+
+ ':tag' -> 'quay.ceph.io/ceph-ci/ceph:tag'
+ 'image:tag' -> 'quay.ceph.io/ceph-ci/image:tag'
+ 'org/image:tag' -> 'quay.ceph.io/org/image:tag'
+ 'example.com/org/image:tag' -> 'example.com/org/image:tag'
+ """
+ try:
+ (image_long, tag) = image.split(':')
+ except ValueError:
+ raise ValueError(f"Container image spec missing tag: {image}") from None
+ domain = 'quay.ceph.io'
+ org = 'ceph-ci'
+ image = 'ceph'
+ image_split = image_long.split('/')
+ assert len(image_split) <= 3
+ match len(image_split):
+ case 3:
+ (domain, org, image) = image_split
+ case 2:
+ (org, image) = image_split
+ case _:
+ if image_split[0]:
+ image = image_split[0]
+ return f"{domain}/{org}/{image}:{tag}"
+
+
+@functools.lru_cache()
+def container_image_exists(image: str):
+ """
+ Use the Quay API to check for the existence of a container image.
+ Only tested with Quay registries.
+ """
+ match = re.match(CONTAINER_REGEXP, image)
+ assert match
+ obj = match.groupdict()
+ url = f"https://{obj['domain']}/api/v1/repository/{obj['org']}/{obj['image']}/tag?filter_tag_name=eq:{obj['tag']}"
+ log.info(f"Checking for container existence at: {url}")
+ resp = requests.get(url)
+ return resp.ok and len(resp.json().get('tags')) >= 1
+
+
+def container_image_for_hash(hash: str, flavor='default', base_image='centos:9'):
+ """
+ Given a sha1 and optionally a base image and flavor, attempt to return a container image locator.
+ """
+ tag = hash
+ if base_image != DEFAULT_CONTAINER_BASE:
+ tag = f"{tag}-{base_image.replace(':', '-')}"
+ if flavor != DEFAULT_CONTAINER_FLAVOR:
+ tag = f"{tag}-{flavor}"
+ image_spec = resolve_container_image(f":{tag}")
+ if container_image_exists(image_spec):
+ return image_spec
+ else:
+ log.error(f"Container image not found for hash '{hash}'")
--- /dev/null
+import pytest
+
+from unittest.mock import patch
+
+from teuthology.util import containers
+
+@pytest.mark.parametrize(
+ 'input, expected',
+ [
+ (':hash', 'quay.ceph.io/ceph-ci/ceph:hash'),
+ ('image:hash', 'quay.ceph.io/ceph-ci/image:hash'),
+ ('org/image:hash', 'quay.ceph.io/org/image:hash'),
+ ('example.com/org/image:hash', 'example.com/org/image:hash'),
+ ('image', ValueError),
+ ('org/image', ValueError),
+ ('domain.net/org/image', ValueError),
+ ]
+)
+def test_resolve_container_image(input, expected):
+ if isinstance(expected, str):
+ assert expected == containers.resolve_container_image(input)
+ else:
+ with pytest.raises(expected):
+ containers.resolve_container_image(input)
+
+@pytest.mark.parametrize(
+ 'image, url',
+ [
+ ('example.com/org/image:tag', 'https://example.com/api/v1/repository/org/image/tag?filter_tag_name=eq:tag'),
+ ]
+)
+def test_container_image_exists(image, url):
+ with patch("teuthology.util.containers.requests.get") as m_get:
+ containers.container_image_exists(image)
+ m_get.assert_called_once_with(url)
+
+
+@pytest.mark.parametrize(
+ 'hash, flavor, base_image, rci_input',
+ [
+ ('hash', 'flavor', 'base-image', ':hash-base-image-flavor'),
+ ('hash', 'default', 'centos:9', ':hash'),
+ ('hash', 'default', 'rockylinux-10', ':hash-rockylinux-10'),
+ ]
+)
+def test_container_image_for_hash(hash, flavor, base_image, rci_input):
+ with patch('teuthology.util.containers.resolve_container_image') as m_rci:
+ with patch('teuthology.util.containers.container_image_exists'):
+ containers.container_image_for_hash(hash, flavor, base_image)
+ m_rci.assert_called_once_with(rci_input)
projects / teuthology.git / commitdiff