'httpd': {'deb': 'apache2', 'rpm': 'httpd'}
}
+# Matches ceph-dev-pipeline ceph_release_repo_template (ceph.repo in ceph-release RPM).
+_PULP_RPM_REPO_TEMPLATE = '''\
+[Ceph]
+name=Ceph packages for $basearch
+baseurl={repo_base_url}/$basearch
+enabled=1
+gpgcheck=0
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/autobuild.asc
+
+[Ceph-noarch]
+name=Ceph noarch packages
+baseurl={repo_base_url}/noarch
+enabled=1
+gpgcheck=0
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/autobuild.asc
+
+[ceph-source]
+name=Ceph source packages
+baseurl={repo_base_url}/SRPMS
+enabled=1
+gpgcheck=0
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/autobuild.asc
+'''
+
+# Apt sources line for Pulp deb distributions (unsigned repo metadata).
+# Suite/components are "default" / "all" at the distribution base URL.
+_PULP_DEB_REPO_TEMPLATE = 'deb [trusted=yes] {repo_base_url}/ default all\n'
+
def get_package_name(pkg, rem):
"""
)
+def _package_source_config(job_config, source):
+ """
+ Merge site-wide package source settings from teuthology config with any
+ per-job overrides in job_config.
+ """
+ merged = dict(getattr(config, source, None) or {})
+ merged.update(job_config.get(source, {}))
+ return merged
+
+
class ShamanProject(GitbuilderProject):
def __init__(self, project, job_config, ctx=None, remote=None):
super(ShamanProject, self).__init__(project, job_config, ctx, remote)
- self.query_url = 'https://%s/api/' % config.shaman_host
+ shaman_cfg = _package_source_config(job_config, 'shaman')
+ self.query_url = shaman_cfg.get(
+ 'endpoint', 'https://shaman.ceph.com/api/')
+ if not self.query_url.endswith('/'):
+ self.query_url += '/'
# Force to use the "noarch" instead to build the uri.
- self.force_noarch = self.job_config.get("shaman", {}).get("force_noarch", False)
+ self.force_noarch = shaman_cfg.get('force_noarch', False)
def _get_base_url(self):
self.assert_result()
)
+class PulpProject(GitbuilderProject):
+ def __init__(self, project, job_config, ctx=None, remote=None):
+ super(PulpProject, self).__init__(project, job_config, ctx, remote)
+
+ pulp_cfg = _package_source_config(job_config, 'pulp')
+ self.pulp_server_url = pulp_cfg.get(
+ 'endpoint', 'https://pulp.example.com/pulp/api/v3/')
+ if not self.pulp_server_url.endswith('/'):
+ self.pulp_server_url += '/'
+ self.pulp_username = pulp_cfg.get('username')
+ self.pulp_password = pulp_cfg.get('password')
+
+ if not self.pulp_username or not self.pulp_password:
+ raise ValueError("Pulp username and password are required")
+
+ # Force to use the "noarch" instead to build the uri.
+ self.force_noarch = pulp_cfg.get('force_noarch', False)
+
+ @property
+ def _search_uri(self):
+ """Build the search url"""
+ tail = 'rpm' if self.pkg_type == 'rpm' else 'apt'
+ path = f'distributions/{self.pkg_type}/{tail}'
+ return urljoin(self.pulp_server_url, path)
+
+ @property
+ def _result(self):
+ """Get the results from the pulp api"""
+ if getattr(self, '_result_obj', None) is None:
+ # Get the results from the pulp api.
+ self._result_obj = self._search().get('results', [])
+
+ # Check if there is exactly one result.
+ if not len(self._result_obj):
+ log.error(f'No results found for {self._search_uri}')
+ raise VersionNotFoundError(f'No results found for {self._search_uri}')
+ elif len(self._result_obj) > 1:
+ log.error(f'Multiple results found for {self._search_uri}')
+ raise VersionNotFoundError(f'Multiple results found for {self._search_uri}')
+
+ return self._result_obj[0]
+
+ @property
+ def repo_url(self):
+ """Get the repo url from the pulp api"""
+ return urljoin(self.pulp_server_url, self._result.get('base_url', ''))
+
+ @property
+ def build_complete(self):
+ """Check if the build is complete"""
+ return self._result.get('build_complete', True)
+
+ def _get_base_url(self):
+ """Get the base url from the pulp api"""
+ return urljoin(
+ self.pulp_server_url,
+ "/".join(self._result.get('base_url', '').split('/')[:-2])
+ )
+
+ def _search(self):
+ """Search for the package in the pulp api"""
+ # Build the search parameters.
+ labels = f'project={self.project},'
+ labels += f'flavors={self.flavor},'
+ labels += f'distro={self.os_type},'
+ labels += f'distro_version={self.os_version},'
+
+ # Add the architecture to the search parameters.
+ arch = 'noarch' if self.force_noarch else self.arch
+ labels += f'arch={arch},'
+
+ # Add the reference to the search parameters.
+ ref_name, ref_val = list(self._choose_reference().items())[0]
+ labels += f'{ref_name}={ref_val}'
+ resp = requests.get(
+ self._search_uri,
+ params={'pulp_label_select': labels},
+ auth=(self.pulp_username, self.pulp_password)
+ )
+ if not resp.ok:
+ log.error(f'Failed to get packages with labels: {labels}')
+ raise VersionNotFoundError(f'Failed to get packages with labels: {labels}')
+
+ return resp.json()
+
+ @classmethod
+ def _get_distro(cls, distro=None, version=None, codename=None):
+ if distro in ('centos', 'rhel'):
+ distro = 'centos'
+ version = cls._parse_version(version)
+ if distro in ('alma', 'rocky'):
+ version = cls._parse_version(version)
+ if distro in ('ubuntu', 'debian'):
+ version = codename or version
+ return f'{distro}/{version}'
+
+ def _get_package_sha1(self):
+ """Get the package sha1 from the pulp api"""
+ return self._result.get('pulp_labels', {}).get('sha1', None)
+
+ def _get_package_version(self):
+ """Get the package version from the pulp api"""
+ return self._result.get('pulp_labels', {}).get('version', None)
+
+ def _get_rpm_repo_content(self):
+ """
+ yum/dnf repo file for Pulp. Uses the flavor base URL so nodes do not rely on
+ ceph-release RPMs that may still point at Chacra.
+ """
+ repo_base = self.base_url.rstrip('/')
+ return _PULP_RPM_REPO_TEMPLATE.format(repo_base_url=repo_base)
+
+ def _install_rpm_repo(self):
+ dist_release = self.dist_release
+ if dist_release in ['opensuse', 'sle']:
+ return super()._install_rpm_repo()
+
+ repo = self._get_rpm_repo_content()
+ repo_path = '/etc/yum.repos.d/{proj}.repo'.format(proj=self.project)
+ log.info('Writing yum repo from Pulp base %s:\n%s', self.base_url, repo)
+ sudo_write_file(self.remote, repo_path, repo)
+
+ def _remove_rpm_repo(self):
+ if self.dist_release in ['opensuse', 'sle']:
+ return super()._remove_rpm_repo()
+ self.remote.run(
+ args=[
+ 'sudo',
+ 'rm', '-f',
+ '/etc/yum.repos.d/{proj}.repo'.format(proj=self.project),
+ ]
+ )
+
+ def _get_deb_repo_base_url(self):
+ """
+ Apt repository root from the Pulp deb distribution (includes arch, e.g.
+ .../flavors/default/x86_64). Unlike RPM, do not strip arch from base_url.
+ """
+ return self.repo_url.rstrip('/')
+
+ def _get_deb_repo_content(self):
+ """
+ Apt sources.list entry for Pulp using the distribution base URL.
+ """
+ repo_base = self._get_deb_repo_base_url()
+ return _PULP_DEB_REPO_TEMPLATE.format(repo_base_url=repo_base)
+
+ def _install_deb_repo(self):
+ repo = self._get_deb_repo_content()
+ repo_path = '/etc/apt/sources.list.d/{proj}.list'.format(proj=self.project)
+ log.info(
+ 'Writing apt repo from Pulp base %s:\n%s',
+ self._get_deb_repo_base_url(),
+ repo,
+ )
+ sudo_write_file(self.remote, repo_path, repo)
+
+ def _remove_deb_repo(self):
+ self.remote.run(
+ args=[
+ 'sudo',
+ 'rm', '-f',
+ '/etc/apt/sources.list.d/{proj}.list'.format(proj=self.project),
+ ]
+ )
+
+
def get_builder_project():
"""
- Depending on whether config.use_shaman is True or False, return
- GitbuilderProject or ShamanProject (the class, not an instance).
+ Depending on whether config.package_source is 'shaman' or 'pulp', return
+ GitbuilderProject, ShamanProject or PulpProject (the class, not an instance).
"""
- if config.use_shaman is True:
+ if config.package_source == 'shaman':
builder_class = ShamanProject
+ elif config.package_source == 'pulp':
+ builder_class = PulpProject
+ log.info(f'Using pulp project: {config.package_source}')
else:
builder_class = GitbuilderProject
return builder_class
import pytest
-from unittest.mock import patch, Mock
+from unittest.mock import patch, Mock, PropertyMock
from teuthology import packaging
from teuthology.exceptions import VersionNotFoundError
packaging._get_response("google.com", sleep=1, tries=2)
assert m_get.call_count == 1
+ @patch('teuthology.packaging.config')
+ def test_get_builder_project_pulp(self, m_config):
+ m_config.package_source = 'pulp'
+ assert packaging.get_builder_project() is packaging.PulpProject
+
class TestBuilderProject(object):
klass = None
def setup_method(self):
self.p_config = patch('teuthology.packaging.config')
self.m_config = self.p_config.start()
- self.m_config.use_shaman = True
- self.m_config.shaman_host = 'shaman.ceph.com'
+ self.m_config.package_source = 'shaman'
+ self.m_config.shaman = {'endpoint': 'https://shaman.ceph.com/api/'}
self.p_get_config_value = \
patch('teuthology.packaging._get_config_value_for_remote')
self.m_get_config_value = self.p_get_config_value.start()
)
def test_get_distro_config(self, matrix_index):
super().test_get_distro_config(matrix_index)
+
+
+class TestPulpProject(TestBuilderProject):
+ klass = packaging.PulpProject
+
+ def setup_method(self):
+ self.p_config = patch('teuthology.packaging.config')
+ self.m_config = self.p_config.start()
+ self.m_config.package_source = 'pulp'
+ self.m_config.pulp = {
+ 'endpoint': 'https://pulp.example.com/pulp/api/v3/',
+ 'username': 'u',
+ 'password': 'p',
+ }
+ self.p_get_config_value = \
+ patch('teuthology.packaging._get_config_value_for_remote')
+ self.m_get_config_value = self.p_get_config_value.start()
+ self.m_get_config_value.return_value = None
+ self.p_get = patch('requests.get')
+ self.m_get = self.p_get.start()
+
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{
+ 'base_url': 'dist/ceph/main/sha/centos/8/flavors/default/',
+ 'pulp_labels': {
+ 'sha1': 'the_sha1',
+ 'version': '0.90.0',
+ },
+ }],
+ }
+ self.m_get.return_value = resp
+
+ def teardown_method(self):
+ self.p_config.stop()
+ self.p_get_config_value.stop()
+ self.p_get.stop()
+
+ def test_init_requires_pulp_credentials(self):
+ self.m_config.pulp = {'username': '', 'password': 'p'}
+ with pytest.raises(ValueError, match='Pulp username and password are required'):
+ packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'),
+ remote=self._get_remote())
+
+ self.m_config.pulp = {'username': 'u', 'password': ''}
+ with pytest.raises(ValueError, match='Pulp username and password are required'):
+ packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'),
+ remote=self._get_remote())
+
+ def test_init_from_remote_base_url(self):
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.PulpProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestPulpProject, self).test_init_from_remote_base_url(
+ 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
+ )
+
+ def test_init_from_remote_base_url_debian(self):
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.PulpProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestPulpProject, self).test_init_from_remote_base_url_debian(
+ 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
+ )
+
+ def test_init_from_config_base_url(self):
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.PulpProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestPulpProject, self).test_init_from_config_base_url(
+ 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
+ )
+
+ @patch('teuthology.packaging.PulpProject._get_package_sha1')
+ def test_init_from_config_tag_ref(self, m_get_package_sha1):
+ m_get_package_sha1.return_value = 'the_sha1'
+ super(TestPulpProject, self).test_init_from_config_tag_ref()
+
+ def test_init_from_config_tag_overrides_branch_ref(self, caplog):
+ obj = super(TestPulpProject, self)\
+ .test_init_from_config_tag_overrides_branch_ref(caplog)
+ obj._search()
+ labels = self.m_get.call_args[1]['params']['pulp_label_select']
+ assert 'tag=v10.0.1' in labels
+ assert 'branch=jewel' not in labels
+
+ def test_init_from_config_branch_overrides_sha1(self, caplog):
+ obj = super(TestPulpProject, self)\
+ .test_init_from_config_branch_overrides_sha1(caplog)
+ obj._search()
+ labels = self.m_get.call_args[1]['params']['pulp_label_select']
+ assert 'branch=jewel' in labels
+ assert 'sha1=sha1' not in labels
+
+ def test_get_package_version_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{
+ 'base_url': 'a/b/c/d',
+ 'pulp_labels': {'version': '0.90.0'},
+ }],
+ }
+ self.m_get.return_value = resp
+ super(TestPulpProject, self).test_get_package_version_found()
+
+ def test_get_package_version_not_found(self):
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ resp = Mock()
+ resp.ok = False
+ self.m_get.return_value = resp
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ with pytest.raises(VersionNotFoundError):
+ gp.version
+
+ def test_get_package_sha1_fetched_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {'sha1': 'the_sha1'}}],
+ }
+ self.m_get.return_value = resp
+ super(TestPulpProject, self).test_get_package_sha1_fetched_found()
+
+ def test_get_package_sha1_fetched_not_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {}}],
+ }
+ self.m_get.return_value = resp
+ super(TestPulpProject, self).test_get_package_sha1_fetched_not_found()
+
+ DISTRO_MATRIX = [
+ ('rhel', '7.0', None, 'centos/7'),
+ ('alma', '9.7', None, 'alma/9'),
+ ('rocky', '10.1', None, 'rocky/10'),
+ ('centos', '8.1', None, 'centos/8'),
+ ('fedora', '20', None, 'fedora/20'),
+ ('ubuntu', '20.04', 'focal', 'ubuntu/focal'),
+ ]
+
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX)),
+ )
+ def test_get_distro_remote(self, matrix_index):
+ super().test_get_distro_remote(matrix_index)
+
+ DISTRO_MATRIX_NOVER = [
+ ('rhel', None, None, 'centos/8'),
+ ('centos', None, None, 'centos/9'),
+ ('fedora', None, None, 'fedora/25'),
+ ('ubuntu', None, None, 'ubuntu/jammy'),
+ ('alma', None, None, 'alma/9'),
+ ('rocky', None, None, 'rocky/9'),
+ ]
+
+ DISTRO_MATRIX_CONFIG = [
+ ('rhel', '7.0', None, 'centos/7'),
+ ('alma', '9.7', None, 'alma/9'),
+ ('rocky', '10.1', None, 'rocky/10'),
+ ('centos', '8.1', None, 'centos/8'),
+ ('fedora', '20', None, 'fedora/20'),
+ ('ubuntu', '20.04', None, 'ubuntu/focal'),
+ ]
+
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX_CONFIG) + len(DISTRO_MATRIX_NOVER)),
+ )
+ def test_get_distro_config(self, matrix_index):
+ (distro, version, _, expected) = (
+ self.DISTRO_MATRIX_CONFIG + self.DISTRO_MATRIX_NOVER
+ )[matrix_index]
+ config = dict(
+ os_type=distro,
+ os_version=version
+ )
+ gp = self.klass("ceph", config)
+ assert gp.distro == expected
+
+ def test_get_rpm_repo_content_uses_pulp_base_url(self):
+ pulp_base = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'abc123/rocky/10/flavors/default'
+ )
+ rem = self._get_remote()
+ with patch(
+ 'teuthology.packaging.PulpProject.base_url',
+ new_callable=PropertyMock,
+ return_value=pulp_base,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ content = gp._get_rpm_repo_content()
+ assert f'baseurl={pulp_base}/$basearch' in content
+ assert f'baseurl={pulp_base}/noarch' in content
+ assert f'baseurl={pulp_base}/SRPMS' in content
+ assert 'chacra' not in content
+
+ @patch('teuthology.packaging.sudo_write_file')
+ def test_install_rpm_repo_writes_repo_file(self, m_sudo_write_file):
+ pulp_base = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'abc123/rocky/10/flavors/default'
+ )
+ rem = self._get_remote()
+ with patch(
+ 'teuthology.packaging.PulpProject.base_url',
+ new_callable=PropertyMock,
+ return_value=pulp_base,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ gp._install_rpm_repo()
+ m_sudo_write_file.assert_called_once()
+ assert m_sudo_write_file.call_args[0][1] == '/etc/yum.repos.d/ceph.repo'
+ assert pulp_base in m_sudo_write_file.call_args[0][2]
+
+ def test_get_deb_repo_content_uses_pulp_distribution_url(self):
+ pulp_dist = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'ff309bba/ubuntu/jammy/flavors/default/x86_64/'
+ )
+ rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy')
+ with patch(
+ 'teuthology.packaging.PulpProject.repo_url',
+ new_callable=PropertyMock,
+ return_value=pulp_dist,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ content = gp._get_deb_repo_content()
+ assert content == (
+ 'deb [trusted=yes] '
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'ff309bba/ubuntu/jammy/flavors/default/x86_64/ default all\n'
+ )
+
+ @patch('teuthology.packaging.sudo_write_file')
+ def test_install_deb_repo_writes_sources_list(self, m_sudo_write_file):
+ pulp_dist = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'ff309bba/ubuntu/jammy/flavors/default/x86_64/'
+ )
+ rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy')
+ with patch(
+ 'teuthology.packaging.PulpProject.repo_url',
+ new_callable=PropertyMock,
+ return_value=pulp_dist,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ gp._install_deb_repo()
+ m_sudo_write_file.assert_called_once()
+ assert m_sudo_write_file.call_args[0][1] == '/etc/apt/sources.list.d/ceph.list'
+ assert '[trusted=yes]' in m_sudo_write_file.call_args[0][2]
+ assert '/x86_64/ default all' in m_sudo_write_file.call_args[0][2]
projects / teuthology.git / commitdiff