Add custom repos before installing rpm packages on test nodes.
Repository can be specified as a NAME:URL pair. Several repos
can be provided by specifying the option multiple times.
For example,
--test-repo foo:http://example.com/repo/foo \
--test-repo bar:http://example.com/repo/bar
gives two test package repositories named "foo" and "bar".
Conflicts:
scripts/openstack.py
teuthology/openstack/__init__.py
teuthology/orchestra/opsys.py
teuthology/task/install/rpm.py
help='Public facing URL where archives are uploaded',
default='http://teuthology-logs.public.ceph.com',
)
+ parser.add_argument(
+ '--test-repo',
+ action='append',
+ help=('Package repository, or repositories, to be added on test nodes. '
+ 'Repository to be specified as a NAME:URL pair. Multiple '
+ 'repositories can be provided with multiple usage. '
+ 'For example --test-repo foo:http://example.com/repo/foo '
+ '--test-repo bar:http://example.com/repo/bar specifies two '
+ 'test package repositories named "foo" and "bar", respectively.'),
+ default=None,
+ )
return parser
def get_parser():
import teuthology
import time
import types
+import yaml
from subprocess import CalledProcessError
os.environ['OS_TOKEN_VALUE'] = OpenStack.token
OpenStack.token_expires = int(time.time() + OpenStack.token_cache_duration)
os.environ['OS_TOKEN_EXPIRES'] = str(OpenStack.token_expires)
- log.info("caching OS_TOKEN_VALUE "
- "during %s seconds" % OpenStack.token_cache_duration)
+ log.debug("caching OS_TOKEN_VALUE "
+ "during %s seconds" % OpenStack.token_cache_duration)
return True
def get_os_url(self, cmd, type=None):
if original_argv[0] in ('--name',
'--teuthology-branch',
'--teuthology-git-url',
+ '--test-repo',
'--archive-upload',
'--archive-upload-url',
'--key-name',
original_argv.pop(0)
else:
argv.append(original_argv.pop(0))
+ if self.args.test_repo:
+ repos = [{'name':k, 'url': v}
+ for k, v in [x.split(':', 1)
+ for x in self.args.test_repo]]
+ log.info("Using repos: %s" % self.args.test_repo)
+
+ overrides = {
+ 'overrides': {
+ 'install': {
+ 'repos' : repos
+ }
+ }
+ }
+ yaml_data = yaml.dump(overrides, default_flow_style=False)
+ with tempfile.NamedTemporaryFile(mode='w+b',
+ suffix='-artifact.yaml',
+ delete=False) as f:
+ yaml_file = f.name
+ log.debug("Using file " + yaml_file)
+ print >> f, yaml_data
+
+ path = self._upload_yaml_file(yaml_file)
+ argv.append(path)
+
#
# If --upload, provide --archive-upload{,-url} regardless of
# what was originally provided on the command line because the
" ~/.teuthology.yaml"
).format(opt='ceph_git_url', value=ceph_repo)
self.ssh(command)
- argv.append('/home/' + self.username +
- '/teuthology/teuthology/openstack/openstack.yaml')
+ user_home = '/home/' + self.username
+ openstack_home = user_home + '/teuthology/teuthology/openstack'
+ if self.args.test_repo:
+ argv.append(openstack_home + '/openstack-basic.yaml')
+ else:
+ argv.append(openstack_home + '/openstack-basic.yaml')
+ argv.append(openstack_home + '/openstack-buildpackages.yaml')
command = (
"source ~/.bashrc_teuthology ; " + self.teuthology_suite + " " +
" --machine-type openstack " +
--- /dev/null
+overrides:
+ ceph:
+ conf:
+ global:
+ osd heartbeat grace: 100
+ # this line to address issue #1017
+ mon lease: 15
+ mon lease ack timeout: 25
+ s3tests:
+ idle_timeout: 1200
+archive-on-error: true
--- /dev/null
+tasks:
+ - buildpackages:
+ good_machine:
+ disk: 100 # GB
+ ram: 15000 # MB
+ cpus: 16
+ min_machine:
+ disk: 100 # GB
+ ram: 8000 # MB
+ cpus: 1
+++ /dev/null
-overrides:
- ceph:
- conf:
- global:
- osd heartbeat grace: 100
- # this line to address issue #1017
- mon lease: 15
- mon lease ack timeout: 25
- s3tests:
- idle_timeout: 1200
-archive-on-error: true
},
"sle": {
"12.2": "sle",
+ "12.3": "sle",
},
"opensuse-leap": {
"42.2": "leap",
fedora="25",
centos="7.4",
opensuse="42.1",
- sle="12.2",
+ sle="12.3",
rhel="7.4",
debian='8.0'
)
@staticmethod
def _version_to_codename(name, version):
for (_version, codename) in DISTRO_CODENAME_MAP[name].iteritems():
- if version == _version or version.split('.')[0] == _version:
+ if str(version) == _version or str(version).split('.')[0] == _version:
return codename
@staticmethod
if not version:
version = DEFAULT_OS_VERSION.get(self.os_type)
- return version
+ return str(version)
def _get_uri_reference(self):
"""
url = "{base_url}/{arch}".format(
base_url=self.base_url, arch=self.arch)
self.remote.run(args=[
- 'sudo', 'zypper', '-n', 'addrepo', '-p', '1', url, 'ceph-rpm-under-test',
- Raw(';'),
- 'sudo', 'zypper', '-n', '--no-gpg-checks', 'refresh'
+ 'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck',
+ '-p', '1', url, 'ceph-rpm-under-test',
])
else:
self.remote.run(args=['sudo', 'yum', '-y', 'install', url])
def get_initial_tasks(lock, config, machine_type):
- init_tasks = [
- {'internal.check_packages': None},
- {'internal.buildpackages_prep': None},
- ]
+ init_tasks = []
+ overrides = config.get('overrides')
+ having_repos = ('install' in config and 'repos' in config.get('install')) \
+ or (overrides and 'install' in overrides and 'repos' in overrides.get('install'))
+ if not having_repos:
+ init_tasks += [
+ {'internal.check_packages': None},
+ {'internal.buildpackages_prep': None},
+ ]
if 'roles' in config and lock:
msg = ('You cannot specify targets in a config file when using the ' +
'--lock option')
if config.get("extras"):
log.info("Skipping version verification...")
return True
+ if 'repos' in config and config.get('repos'):
+ log.info("Skipping version verification because we have custom repos...")
+ return True
builder = _get_builder_project(ctx, remote, config)
version = builder.version
pkg_to_check = builder.project
'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
])
-
def install_packages(ctx, pkgs, config):
"""
Installs packages on each remote in ctx.
if overrides:
install_overrides = overrides.get('install', {})
teuthology.deep_merge(config, install_overrides.get(project, {}))
+ repos = install_overrides.get('repos', None)
+ log.debug('INSTALL overrides: %s' % install_overrides)
log.debug('config %s' % config)
rhbuild = None
with contextutil.nested(*nested_tasks):
yield
else:
- with contextutil.nested(
- lambda: install(ctx=ctx, config=dict(
+ nested_config = dict(
branch=config.get('branch'),
tag=config.get('tag'),
sha1=config.get('sha1'),
wait_for_package=config.get('wait_for_package', False),
project=project,
packages=config.get('packages', dict()),
- )),
+ )
+ if repos:
+ nested_config['repos'] = repos
+ with contextutil.nested(
+ lambda: install(ctx=ctx, config=nested_config),
lambda: ship_utilities(ctx=ctx, config=None),
):
yield
:param remote: the teuthology.orchestra.remote.Remote object
:param rpm: list of packages names to remove
"""
- rpm = _package_overrides(rpm, remote.os)
+ remote_os = remote.os
+ dist_release = remote_os.name
+ rpm = _package_overrides(rpm, remote_os)
log.info("Removing packages: {pkglist} on rpm system.".format(
pkglist=", ".join(rpm)))
- builder = _get_builder_project(ctx, remote, config)
- dist_release = builder.dist_release
+ repos = config.get('repos')
if dist_release in ['opensuse', 'sle']:
- pkg_mng_cmd = 'zypper'
- pkg_mng_opts = '-n --no-gpg-checks'
- pkg_mng_subcommand_opts = '--capability'
+ remote.run(args='''
+ for d in {rpms} ; do
+ sudo zypper -n --no-gpg-checks remove --capability $d || true
+ done'''.format(rpms=' '.join(rpm)))
+ remote.run(args='sudo zypper clean -a')
else:
- pkg_mng_cmd = 'yum'
- pkg_mng_opts = '-y'
- pkg_mng_subcommand_opts = ''
+ remote.run(args='''
+ for d in {rpms} ; do
+ sudo yum -y remove $d || true
+ done'''.format(rpms=' '.join(rpm)))
+ remote.run(args='sudo yum clean all')
+
+ if repos:
+ if dist_release in ['opensuse', 'sle']:
+ _zypper_removerepo(remote, repos)
+ else:
+ raise Exception('Custom repos were specified for %s ' % remote_os +
+ 'but these are currently not supported')
+ else:
+ builder = _get_builder_project(ctx, remote, config)
+ builder.remove_repo()
- remote.run(
- args=[
- 'for', 'd', 'in',
- ] + rpm + [
- run.Raw(';'),
- 'do',
- 'sudo',
- pkg_mng_cmd, pkg_mng_opts, 'remove', pkg_mng_subcommand_opts,
- run.Raw('$d'), run.Raw('||'), 'true', run.Raw(';'),
- 'done',
- ])
if dist_release in ['opensuse', 'sle']:
- pkg_mng_opts = '-a'
+ remote.run(args='sudo zypper clean -a')
else:
- pkg_mng_opts = 'all'
- remote.run(
- args=[
- 'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts,
- ])
-
- builder.remove_repo()
-
- if dist_release not in ['opensuse', 'sle']:
- pkg_mng_opts = 'expire-cache'
- remote.run(
- args=[
- 'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts,
- ])
+ remote.run(args='sudo yum clean expire-cache')
def _package_overrides(pkgs, os):
result.append(pkg)
return result
+def _zypper_addrepo(remote, repo_list):
+ """
+ Add zypper repos to the remote system.
+
+ :param remote: remote node where to add packages
+ :param repo_list: list of dictionaries with keys 'name', 'url'
+ :return:
+ """
+ for repo in repo_list:
+ remote.run(args=[
+ 'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck',
+ '-p', '1', repo['url'], repo['name'],
+ ])
+
+def _zypper_removerepo(remote, repo_list):
+ """
+ Remove zypper repos on the remote system.
+
+ :param remote: remote node where to remove packages from
+ :param repo_list: list of dictionaries with keys 'name', 'url'
+ :return:
+ """
+ for repo in repo_list:
+ remote.run(args=[
+ 'sudo', 'zypper', '-n', 'removerepo', repo['name'],
+ ])
def _update_package_list_and_install(ctx, remote, rpm, config):
"""
rpm += system_pkglist.get('rpm')
else:
rpm += system_pkglist
- rpm = _package_overrides(rpm, remote.os)
- builder = _get_builder_project(ctx, remote, config)
- log.info('Pulling from %s', builder.base_url)
- log.info('Package version is %s', builder.version)
+ remote_os = remote.os
+ rpm = _package_overrides(rpm, remote_os)
log.info("Installing packages: {pkglist} on remote rpm {arch}".format(
- pkglist=", ".join(rpm), arch=builder.arch))
- builder.install_repo()
+ pkglist=", ".join(rpm), arch=remote.arch))
+
+ dist_release = remote_os.name
+ repos = config.get('repos')
+ if repos:
+ log.debug("Adding repos: %s" % repos)
+ if dist_release in ['opensuse', 'sle']:
+ _zypper_addrepo(remote, repos)
+ else:
+ raise Exception('Custom repos were specified for %s ' % remote_os +
+ 'but these are currently not supported')
+ else:
+ builder = _get_builder_project(ctx, remote, config)
+ log.info('Pulling from %s', builder.base_url)
+ log.info('Package version is %s', builder.version)
+ builder.install_repo()
- dist_release = builder.dist_release
- project = builder.project
if dist_release not in ['opensuse', 'sle']:
+ project = builder.project
uri = builder.uri_reference
_yum_fix_repo_priority(remote, project, uri)
_yum_fix_repo_host(remote, project)
# in the job config.
if os_type and sha1:
package = get_builder_project()("ceph", ctx.config)
- template = "Checking packages for os_type,'{os}' flavor '{flav}' and" \
- " ceph hash '{ver}'"
+ template = "Checking packages for os_type '{os}', " \
+ "flavor '{flav}' and ceph hash '{ver}'"
log.info(
template.format(
os=package.os_type,
projects / teuthology.git / commitdiff