From: Vaibhav Mahajan Date: Wed, 25 Mar 2026 14:18:44 +0000 (+0530) Subject: feature: add `PulpProject` class for Pulp-based package repository support X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0e71339b384e4f0248751e9259581abe4ea0cbcf;p=teuthology.git feature: add `PulpProject` class for Pulp-based package repository support Add a `PulpProject` subclass of `GitbuilderProject` so teuthology can use Pulp-hosted package repos when `config.package_source` is `pulp`, alongside the existing `Gitbuilder` and `Shaman` artifact backends. Signed-off-by: Vaibhav Mahajan --- diff --git a/docs/detailed_test_config.rst b/docs/detailed_test_config.rst index 2a46c4d77..7e1d200fd 100644 --- a/docs/detailed_test_config.rst +++ b/docs/detailed_test_config.rst @@ -296,14 +296,23 @@ specified in ``$HOME/.teuthology.yaml``:: test_path: -Shaman options -============== +Package source options +====================== + +Package source options can be specified in the top-level configuration, like:: + +package_source: shaman | pulp + +Options for shaman: -Shaman is a helper class which could be used to build the uri for specified -packages based the 'shaman_host': 'shaman.ceph.com'. +shaman: + endpoint: "https://shaman.ceph.com/api/" + force_noarch: True -Options:: +Options for pulp: - use_shaman: True # Enable to use Shaman, False as default - shaman: - force_noarch: True # Force to use "noarch" to build the uri +pulp: + endpoint: "https://pulp.front.sepia.ceph.com/pulp/api/v3/" + username: pulp-username + password: pulp-password + force_noarch: True \ No newline at end of file diff --git a/docs/siteconfig.rst b/docs/siteconfig.rst index ee5d18fa5..8912e19cc 100644 --- a/docs/siteconfig.rst +++ b/docs/siteconfig.rst @@ -269,3 +269,18 @@ Here is a sample configuration with many of the options set and documented:: # Do not allow more than that many jobs in a single run by default. # To disable this check use 0. job_threshold: 500 + + # Setting for artifacts manager (shaman or pulp) + package_source: shaman | pulp + + # Settings for shaman package manager (https://shaman.ceph.com/) + shaman: + endpoint: "https://shaman.ceph.com/api/" + force_noarch: True + + # Settings for pulp package manager (https://pulpproject.org/) + pulp: + endpoint: "https://pulp.example.com/pulp/api/v3/" + username: pulp-username + password: pulp-password + force_noarch: True \ No newline at end of file diff --git a/teuthology/config.py b/teuthology/config.py index 3400959d8..7e51b3dde 100644 --- a/teuthology/config.py +++ b/teuthology/config.py @@ -181,8 +181,11 @@ class TeuthologyConfig(YamlConfig): 'kojiroot_url': 'https://kojipkgs.fedoraproject.org/packages', 'koji_task_url': 'https://kojipkgs.fedoraproject.org/work/', 'baseurl_template': 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}', - 'use_shaman': True, - 'shaman_host': 'shaman.ceph.com', + 'package_source': 'shaman', + 'shaman': { + 'endpoint': 'https://shaman.ceph.com/api/', + }, + 'pulp': {}, 'teuthology_path': None, 'suite_verify_ceph_hash': True, 'suite_allow_missing_packages': False, diff --git a/teuthology/openstack/setup-openstack.sh b/teuthology/openstack/setup-openstack.sh index 526ba9813..26aec69c2 100755 --- a/teuthology/openstack/setup-openstack.sh +++ b/teuthology/openstack/setup-openstack.sh @@ -54,7 +54,7 @@ function create_config() { cat > ~/.teuthology.yaml < 1: + log.error(f'Multiple results found for {self._search_uri}') + raise VersionNotFoundError(f'Multiple results found for {self._search_uri}') + + return self._result_obj[0] + + @property + def repo_url(self): + """Get the repo url from the pulp api""" + return urljoin(self.pulp_server_url, self._result.get('base_url', '')) + + @property + def build_complete(self): + """Check if the build is complete""" + return self._result.get('build_complete', True) + + def _get_base_url(self): + """Get the base url from the pulp api""" + return urljoin( + self.pulp_server_url, + "/".join(self._result.get('base_url', '').split('/')[:-2]) + ) + + def _search(self): + """Search for the package in the pulp api""" + # Build the search parameters. + labels = f'project={self.project},' + labels += f'flavors={self.flavor},' + labels += f'distro={self.os_type},' + labels += f'distro_version={self.os_version},' + + # Add the architecture to the search parameters. + arch = 'noarch' if self.force_noarch else self.arch + labels += f'arch={arch},' + + # Add the reference to the search parameters. + ref_name, ref_val = list(self._choose_reference().items())[0] + labels += f'{ref_name}={ref_val}' + resp = requests.get( + self._search_uri, + params={'pulp_label_select': labels}, + auth=(self.pulp_username, self.pulp_password) + ) + if not resp.ok: + log.error(f'Failed to get packages with labels: {labels}') + raise VersionNotFoundError(f'Failed to get packages with labels: {labels}') + + return resp.json() + + @classmethod + def _get_distro(cls, distro=None, version=None, codename=None): + if distro in ('centos', 'rhel'): + distro = 'centos' + version = cls._parse_version(version) + if distro in ('alma', 'rocky'): + version = cls._parse_version(version) + if distro in ('ubuntu', 'debian'): + version = codename or version + return f'{distro}/{version}' + + def _get_package_sha1(self): + """Get the package sha1 from the pulp api""" + return self._result.get('pulp_labels', {}).get('sha1', None) + + def _get_package_version(self): + """Get the package version from the pulp api""" + return self._result.get('pulp_labels', {}).get('version', None) + + def _get_rpm_repo_content(self): + """ + yum/dnf repo file for Pulp. Uses the flavor base URL so nodes do not rely on + ceph-release RPMs that may still point at Chacra. + """ + repo_base = self.base_url.rstrip('/') + return _PULP_RPM_REPO_TEMPLATE.format(repo_base_url=repo_base) + + def _install_rpm_repo(self): + dist_release = self.dist_release + if dist_release in ['opensuse', 'sle']: + return super()._install_rpm_repo() + + repo = self._get_rpm_repo_content() + repo_path = '/etc/yum.repos.d/{proj}.repo'.format(proj=self.project) + log.info('Writing yum repo from Pulp base %s:\n%s', self.base_url, repo) + sudo_write_file(self.remote, repo_path, repo) + + def _remove_rpm_repo(self): + if self.dist_release in ['opensuse', 'sle']: + return super()._remove_rpm_repo() + self.remote.run( + args=[ + 'sudo', + 'rm', '-f', + '/etc/yum.repos.d/{proj}.repo'.format(proj=self.project), + ] + ) + + def _get_deb_repo_base_url(self): + """ + Apt repository root from the Pulp deb distribution (includes arch, e.g. + .../flavors/default/x86_64). Unlike RPM, do not strip arch from base_url. + """ + return self.repo_url.rstrip('/') + + def _get_deb_repo_content(self): + """ + Apt sources.list entry for Pulp using the distribution base URL. + """ + repo_base = self._get_deb_repo_base_url() + return _PULP_DEB_REPO_TEMPLATE.format(repo_base_url=repo_base) + + def _install_deb_repo(self): + repo = self._get_deb_repo_content() + repo_path = '/etc/apt/sources.list.d/{proj}.list'.format(proj=self.project) + log.info( + 'Writing apt repo from Pulp base %s:\n%s', + self._get_deb_repo_base_url(), + repo, + ) + sudo_write_file(self.remote, repo_path, repo) + + def _remove_deb_repo(self): + self.remote.run( + args=[ + 'sudo', + 'rm', '-f', + '/etc/apt/sources.list.d/{proj}.list'.format(proj=self.project), + ] + ) + + def get_builder_project(): """ - Depending on whether config.use_shaman is True or False, return - GitbuilderProject or ShamanProject (the class, not an instance). + Depending on whether config.package_source is 'shaman' or 'pulp', return + GitbuilderProject, ShamanProject or PulpProject (the class, not an instance). """ - if config.use_shaman is True: + if config.package_source == 'shaman': builder_class = ShamanProject + elif config.package_source == 'pulp': + builder_class = PulpProject + log.info(f'Using pulp project: {config.package_source}') else: builder_class = GitbuilderProject return builder_class diff --git a/teuthology/run.py b/teuthology/run.py index be5c2029a..56df844b3 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -390,10 +390,10 @@ def main(args): # fetches the tasks and returns a new suite_path if needed config["suite_path"] = fetch_tasks_if_needed(config) - # If the job has a 'use_shaman' key, use that value to override the global - # config's value. - if config.get('use_shaman') is not None: - teuth_config.use_shaman = config['use_shaman'] + # If the job has a 'package_source' key, use that value to override the global + # config's 'package_source' value. + if config.get('package_source') is not None: + teuth_config.package_source = config['package_source'] #could be refactored for setting and unsetting in hackish way if interactive_on_error: diff --git a/teuthology/task/install/__init__.py b/teuthology/task/install/__init__.py index ebe18d043..0f22c12a2 100644 --- a/teuthology/task/install/__init__.py +++ b/teuthology/task/install/__init__.py @@ -637,6 +637,8 @@ def task(ctx, config): nested_config['repos'] = repos if 'shaman' in config: nested_config['shaman'] = config['shaman'] + if 'pulp' in config: + nested_config['pulp'] = config['pulp'] with contextutil.nested( lambda: install(ctx=ctx, config=nested_config), lambda: ship_utilities(ctx=ctx, config=None), diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index 85a69d917..f525c07c4 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -374,7 +374,7 @@ def download_kernel(ctx, config): ctx=ctx, remote=role_remote, ) - if teuth_config.use_shaman: + if teuth_config.package_source in ['shaman', 'pulp']: if role_remote.os.package_type == 'rpm': arch = builder.arch baseurl = urljoin( @@ -1443,7 +1443,7 @@ def process_role(ctx, config, timeout, role, role_config): ctx.summary['{role}-kernel-sha1'.format(role=role)] = sha1 if need_to_install(ctx, role, sha1): - if teuth_config.use_shaman: + if teuth_config.package_source in ('shaman', 'pulp'): version = builder.scm_version else: version = builder.version diff --git a/teuthology/test/test_packaging.py b/teuthology/test/test_packaging.py index 0ab4a733b..046105e87 100644 --- a/teuthology/test/test_packaging.py +++ b/teuthology/test/test_packaging.py @@ -1,6 +1,6 @@ import pytest -from unittest.mock import patch, Mock +from unittest.mock import patch, Mock, PropertyMock from teuthology import packaging from teuthology.exceptions import VersionNotFoundError @@ -302,6 +302,11 @@ class TestPackaging(object): packaging._get_response("google.com", sleep=1, tries=2) assert m_get.call_count == 1 + @patch('teuthology.packaging.config') + def test_get_builder_project_pulp(self, m_config): + m_config.package_source = 'pulp' + assert packaging.get_builder_project() is packaging.PulpProject + class TestBuilderProject(object): klass = None @@ -533,8 +538,8 @@ class TestShamanProject(TestBuilderProject): def setup_method(self): self.p_config = patch('teuthology.packaging.config') self.m_config = self.p_config.start() - self.m_config.use_shaman = True - self.m_config.shaman_host = 'shaman.ceph.com' + self.m_config.package_source = 'shaman' + self.m_config.shaman = {'endpoint': 'https://shaman.ceph.com/api/'} self.p_get_config_value = \ patch('teuthology.packaging._get_config_value_for_remote') self.m_get_config_value = self.p_get_config_value.start() @@ -810,3 +815,272 @@ class TestShamanProject(TestBuilderProject): ) def test_get_distro_config(self, matrix_index): super().test_get_distro_config(matrix_index) + + +class TestPulpProject(TestBuilderProject): + klass = packaging.PulpProject + + def setup_method(self): + self.p_config = patch('teuthology.packaging.config') + self.m_config = self.p_config.start() + self.m_config.package_source = 'pulp' + self.m_config.pulp = { + 'endpoint': 'https://pulp.example.com/pulp/api/v3/', + 'username': 'u', + 'password': 'p', + } + self.p_get_config_value = \ + patch('teuthology.packaging._get_config_value_for_remote') + self.m_get_config_value = self.p_get_config_value.start() + self.m_get_config_value.return_value = None + self.p_get = patch('requests.get') + self.m_get = self.p_get.start() + + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{ + 'base_url': 'dist/ceph/main/sha/centos/8/flavors/default/', + 'pulp_labels': { + 'sha1': 'the_sha1', + 'version': '0.90.0', + }, + }], + } + self.m_get.return_value = resp + + def teardown_method(self): + self.p_config.stop() + self.p_get_config_value.stop() + self.p_get.stop() + + def test_init_requires_pulp_credentials(self): + self.m_config.pulp = {'username': '', 'password': 'p'} + with pytest.raises(ValueError, match='Pulp username and password are required'): + packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'), + remote=self._get_remote()) + + self.m_config.pulp = {'username': 'u', 'password': ''} + with pytest.raises(ValueError, match='Pulp username and password are required'): + packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'), + remote=self._get_remote()) + + def test_init_from_remote_base_url(self): + def m_get_base_url(obj): + obj._search() + return self.m_get.call_args_list[0][0][0] + with patch( + 'teuthology.packaging.PulpProject._get_base_url', + new=m_get_base_url, + ): + super(TestPulpProject, self).test_init_from_remote_base_url( + 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt', + ) + + def test_init_from_remote_base_url_debian(self): + def m_get_base_url(obj): + obj._search() + return self.m_get.call_args_list[0][0][0] + with patch( + 'teuthology.packaging.PulpProject._get_base_url', + new=m_get_base_url, + ): + super(TestPulpProject, self).test_init_from_remote_base_url_debian( + 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt', + ) + + def test_init_from_config_base_url(self): + def m_get_base_url(obj): + obj._search() + return self.m_get.call_args_list[0][0][0] + with patch( + 'teuthology.packaging.PulpProject._get_base_url', + new=m_get_base_url, + ): + super(TestPulpProject, self).test_init_from_config_base_url( + 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt', + ) + + @patch('teuthology.packaging.PulpProject._get_package_sha1') + def test_init_from_config_tag_ref(self, m_get_package_sha1): + m_get_package_sha1.return_value = 'the_sha1' + super(TestPulpProject, self).test_init_from_config_tag_ref() + + def test_init_from_config_tag_overrides_branch_ref(self, caplog): + obj = super(TestPulpProject, self)\ + .test_init_from_config_tag_overrides_branch_ref(caplog) + obj._search() + labels = self.m_get.call_args[1]['params']['pulp_label_select'] + assert 'tag=v10.0.1' in labels + assert 'branch=jewel' not in labels + + def test_init_from_config_branch_overrides_sha1(self, caplog): + obj = super(TestPulpProject, self)\ + .test_init_from_config_branch_overrides_sha1(caplog) + obj._search() + labels = self.m_get.call_args[1]['params']['pulp_label_select'] + assert 'branch=jewel' in labels + assert 'sha1=sha1' not in labels + + def test_get_package_version_found(self): + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{ + 'base_url': 'a/b/c/d', + 'pulp_labels': {'version': '0.90.0'}, + }], + } + self.m_get.return_value = resp + super(TestPulpProject, self).test_get_package_version_found() + + def test_get_package_version_not_found(self): + rem = self._get_remote() + ctx = dict(foo="bar") + resp = Mock() + resp.ok = False + self.m_get.return_value = resp + gp = self.klass("ceph", {}, ctx=ctx, remote=rem) + with pytest.raises(VersionNotFoundError): + gp.version + + def test_get_package_sha1_fetched_found(self): + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {'sha1': 'the_sha1'}}], + } + self.m_get.return_value = resp + super(TestPulpProject, self).test_get_package_sha1_fetched_found() + + def test_get_package_sha1_fetched_not_found(self): + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {}}], + } + self.m_get.return_value = resp + super(TestPulpProject, self).test_get_package_sha1_fetched_not_found() + + DISTRO_MATRIX = [ + ('rhel', '7.0', None, 'centos/7'), + ('alma', '9.7', None, 'alma/9'), + ('rocky', '10.1', None, 'rocky/10'), + ('centos', '8.1', None, 'centos/8'), + ('fedora', '20', None, 'fedora/20'), + ('ubuntu', '20.04', 'focal', 'ubuntu/focal'), + ] + + @pytest.mark.parametrize( + "matrix_index", + range(len(DISTRO_MATRIX)), + ) + def test_get_distro_remote(self, matrix_index): + super().test_get_distro_remote(matrix_index) + + DISTRO_MATRIX_NOVER = [ + ('rhel', None, None, 'centos/8'), + ('centos', None, None, 'centos/9'), + ('fedora', None, None, 'fedora/25'), + ('ubuntu', None, None, 'ubuntu/jammy'), + ('alma', None, None, 'alma/9'), + ('rocky', None, None, 'rocky/9'), + ] + + DISTRO_MATRIX_CONFIG = [ + ('rhel', '7.0', None, 'centos/7'), + ('alma', '9.7', None, 'alma/9'), + ('rocky', '10.1', None, 'rocky/10'), + ('centos', '8.1', None, 'centos/8'), + ('fedora', '20', None, 'fedora/20'), + ('ubuntu', '20.04', None, 'ubuntu/focal'), + ] + + @pytest.mark.parametrize( + "matrix_index", + range(len(DISTRO_MATRIX_CONFIG) + len(DISTRO_MATRIX_NOVER)), + ) + def test_get_distro_config(self, matrix_index): + (distro, version, _, expected) = ( + self.DISTRO_MATRIX_CONFIG + self.DISTRO_MATRIX_NOVER + )[matrix_index] + config = dict( + os_type=distro, + os_version=version + ) + gp = self.klass("ceph", config) + assert gp.distro == expected + + def test_get_rpm_repo_content_uses_pulp_base_url(self): + pulp_base = ( + 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/' + 'abc123/rocky/10/flavors/default' + ) + rem = self._get_remote() + with patch( + 'teuthology.packaging.PulpProject.base_url', + new_callable=PropertyMock, + return_value=pulp_base, + ): + gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem) + content = gp._get_rpm_repo_content() + assert f'baseurl={pulp_base}/$basearch' in content + assert f'baseurl={pulp_base}/noarch' in content + assert f'baseurl={pulp_base}/SRPMS' in content + assert 'chacra' not in content + + @patch('teuthology.packaging.sudo_write_file') + def test_install_rpm_repo_writes_repo_file(self, m_sudo_write_file): + pulp_base = ( + 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/' + 'abc123/rocky/10/flavors/default' + ) + rem = self._get_remote() + with patch( + 'teuthology.packaging.PulpProject.base_url', + new_callable=PropertyMock, + return_value=pulp_base, + ): + gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem) + gp._install_rpm_repo() + m_sudo_write_file.assert_called_once() + assert m_sudo_write_file.call_args[0][1] == '/etc/yum.repos.d/ceph.repo' + assert pulp_base in m_sudo_write_file.call_args[0][2] + + def test_get_deb_repo_content_uses_pulp_distribution_url(self): + pulp_dist = ( + 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/' + 'ff309bba/ubuntu/jammy/flavors/default/x86_64/' + ) + rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy') + with patch( + 'teuthology.packaging.PulpProject.repo_url', + new_callable=PropertyMock, + return_value=pulp_dist, + ): + gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem) + content = gp._get_deb_repo_content() + assert content == ( + 'deb [trusted=yes] ' + 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/' + 'ff309bba/ubuntu/jammy/flavors/default/x86_64/ default all\n' + ) + + @patch('teuthology.packaging.sudo_write_file') + def test_install_deb_repo_writes_sources_list(self, m_sudo_write_file): + pulp_dist = ( + 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/' + 'ff309bba/ubuntu/jammy/flavors/default/x86_64/' + ) + rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy') + with patch( + 'teuthology.packaging.PulpProject.repo_url', + new_callable=PropertyMock, + return_value=pulp_dist, + ): + gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem) + gp._install_deb_repo() + m_sudo_write_file.assert_called_once() + assert m_sudo_write_file.call_args[0][1] == '/etc/apt/sources.list.d/ceph.list' + assert '[trusted=yes]' in m_sudo_write_file.call_args[0][2] + assert '/x86_64/ default all' in m_sudo_write_file.call_args[0][2]