from teuthology import contextutil
from teuthology.parallel import parallel
from ..orchestra import run
+from ..orchestra.run import CommandFailedError
log = logging.getLogger(__name__)
# Should the RELEASE value get extracted from somewhere?
RELEASE = "1-0"
+# This is intended to be a complete listing of ceph packages. If we're going
+# to hardcode this stuff, I don't want to do it in more than once place.
+PACKAGES = {}
+PACKAGES['ceph'] = {}
+PACKAGES['ceph']['deb'] = [
+ 'ceph',
+ 'ceph-dbg',
+ 'ceph-mds',
+ 'ceph-mds-dbg',
+ 'ceph-common',
+ 'ceph-common-dbg',
+ 'ceph-fuse',
+ 'ceph-fuse-dbg',
+ 'ceph-test',
+ 'ceph-test-dbg',
+ 'radosgw',
+ 'radosgw-dbg',
+ 'python-ceph',
+ 'libcephfs1',
+ 'libcephfs1-dbg',
+ 'libcephfs-java',
+ 'librados2',
+ 'librados2-dbg',
+ 'librbd1',
+ 'librbd1-dbg',
+]
+PACKAGES['ceph']['rpm'] = [
+ 'ceph-debug',
+ 'ceph-radosgw',
+ 'ceph-test',
+ 'ceph-devel',
+ 'ceph',
+ 'ceph-fuse',
+ 'rest-bench',
+ 'libcephfs_jni1',
+ 'libcephfs1',
+ 'python-ceph',
+]
+
+
+def _run_and_log_error_if_fails(remote, args):
+ response = StringIO()
+ try:
+ remote.run(
+ args=args,
+ stdout=response,
+ stderr=response,
+ )
+ except CommandFailedError:
+ log.error(response.getvalue().strip())
+ raise
+
+
def _get_baseurlinfo_and_dist(ctx, remote, config):
retval = {}
relval = None
- r = remote.run(
- args=['lsb_release', '-sc'],
- stdout=StringIO(),
- )
- retval['dist'] = r.stdout.getvalue().strip()
r = remote.run(
args=['arch'],
stdout=StringIO(),
retval['relval'] = r.stdout.getvalue().strip()
dist_name = None
if ((retval['distro'] == 'CentOS') | (retval['distro'] == 'RedHatEnterpriseServer')):
- distri = 'centos'
- dist_name = 'el'
relval = retval['relval']
relval = relval[0:relval.find('.')]
- retval['distro_release'] = '%s%s' %(distri, relval)
- retval['dist_release'] = '%s%s' %(dist_name, relval)
+ distri = 'centos'
+ retval['distro_release'] = '%s%s' % (distri, relval)
+ retval['dist'] = retval['distro_release']
+ dist_name = 'el'
+ retval['dist_release'] = '%s%s' % (dist_name, relval)
elif retval['distro'] == 'Fedora':
distri = retval['distro']
dist_name = 'fc'
- retval['distro_release'] = '%s%s' %(dist_name, retval['relval'])
- retval['dist_release'] = '%s%s' %(dist_name, retval['relval'])
+ retval['distro_release'] = '%s%s' % (dist_name, retval['relval'])
+ retval['dist_release'] = retval['distro_release']
else:
+ r = remote.run(
+ args=['lsb_release', '-sc'],
+ stdout=StringIO(),
+ )
+ retval['dist'] = r.stdout.getvalue().strip()
retval['distro_release'] = None
retval['dist_release'] = None
return retval
+
+def _get_baseurl(ctx, remote, config):
+ # get distro name and arch
+ baseparms = _get_baseurlinfo_and_dist(ctx, remote, config)
+ base_url = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}'.format(
+ host=ctx.teuthology_config.get('gitbuilder_host',
+ 'gitbuilder.ceph.com'),
+ proj=config.get('project', 'ceph'),
+ pkg_type=remote.system_type,
+ **baseparms
+ )
+ return base_url
+
+
+class VersionNotFoundError(Exception):
+ def __init__(self, url):
+ self.url = url
+
+ def __str__(self):
+ return "Failed to fetch package version from %s" % self.url
+
+
+def _block_looking_for_package_version(remote, base_url, wait=False):
+ """
+ Look for, and parse, a file called 'version' in base_url.
+
+ wait -- wait forever for the file to show up. (default False)
+ """
+ while True:
+ r = remote.run(
+ args=['wget', '-q', '-O-', base_url + '/version'],
+ stdout=StringIO(),
+ check_status=False,
+ )
+ if r.exitstatus != 0:
+ if wait:
+ log.info('Package not there yet, waiting...')
+ time.sleep(15)
+ continue
+ raise VersionNotFoundError(base_url)
+ break
+ version = r.stdout.getvalue().strip()
+ return version
+
+
def _update_deb_package_list_and_install(ctx, remote, debs, config):
"""
updates the package list so that apt-get can
stdout=StringIO(),
)
- # get distro name and arch
baseparms = _get_baseurlinfo_and_dist(ctx, remote, config)
log.info("Installing packages: {pkglist} on remote deb {arch}".format(
- pkglist=", ".join(debs), arch=baseparms['arch']))
- dist = baseparms['dist']
- base_url = 'http://{host}/{proj}-deb-{dist}-{arch}-{flavor}/{uri}'.format(
- host=ctx.teuthology_config.get('gitbuilder_host',
- 'gitbuilder.ceph.com'),
- proj=config.get('project', 'ceph'),
- **baseparms
- )
+ pkglist=", ".join(debs), arch=baseparms['arch'])
+ )
+ # get baseurl
+ base_url = _get_baseurl(ctx, remote, config)
log.info('Pulling from %s', base_url)
# get package version string
remote.run(
args=[
- 'echo', 'deb', base_url, dist, 'main',
+ 'echo', 'deb', base_url, baseparms['dist'], 'main',
run.Raw('|'),
'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format(proj=config.get('project', 'ceph')),
],
stdout=StringIO(),
)
+
+def _yum_fix_repo_priority(remote, project):
+ remote.run(
+ args=[
+ 'sudo',
+ 'sed',
+ '-i',
+ run.Raw('\':a;N;$!ba;s/enabled=1\\ngpg/enabled=1\\npriority=1\\ngpg/g\''),
+ '/etc/yum.repos.d/%s.repo' % project,
+ ]
+ )
+
+
def _update_rpm_package_list_and_install(ctx, remote, rpm, config):
baseparms = _get_baseurlinfo_and_dist(ctx, remote, config)
log.info("Installing packages: {pkglist} on remote rpm {arch}".format(
remote.run(args=['rm', '-f', rpm_name])
#Fix Repo Priority
- remote.run(
- args=[
- 'sudo', 'sed', '-i', run.Raw('\':a;N;$!ba;s/enabled=1\\ngpg/enabled=1\\npriority=1\\ngpg/g\''), '/etc/yum.repos.d/ceph.repo',
- ])
+ _yum_fix_repo_priority(remote, config.get('project', 'ceph'))
remote.run(
args=[
if project == 'ceph':
purge_data(ctx)
-def _upgrade_packages(ctx, config, remote, debs, branch):
+
+def _upgrade_deb_packages(ctx, config, remote, debs, branch):
"""
upgrade all packages
"""
r = remote.run(
args=[
'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
- ],
+ ],
stdout=StringIO(),
)
if r.stdout.getvalue().find('Ceph automated package') == -1:
stdout=StringIO(),
)
+
+def _upgrade_rpm_packages(ctx, config, remote, pkgs, branch):
+ """
+ Upgrade RPM packages.
+ """
+ distinfo = _get_baseurlinfo_and_dist(ctx, remote, config)
+ log.info(
+ "Host {host} is: {distro} {ver} {arch}".format(
+ host=remote.shortname,
+ distro=distinfo['distro'],
+ ver=distinfo['relval'],
+ arch=distinfo['arch'],)
+ )
+
+ base_url = _get_baseurl(ctx, remote, config)
+ log.info('Repo base URL: %s', base_url)
+ version = _block_looking_for_package_version(
+ remote,
+ base_url,
+ config.get('wait_for_package', False))
+ # FIXME: 'version' as retreived from the repo is actually the RPM version
+ # PLUS *part* of the release. Example:
+ # Right now, ceph master is given the following version in the repo file:
+ # v0.67-rc3.164.gd5aa3a9 - whereas in reality the RPM version is 0.61.7
+ # and the release is 37.g1243c97.el6 (for centos6).
+ # Point being, I have to mangle a little here.
+ if version[0] == 'v':
+ version = version[1:]
+ version = "{repover}.{therest}".format(
+ repover=version,
+ therest=distinfo['dist_release'],
+ )
+
+ project = config.get('project', 'ceph')
+
+ # Remove the -release package before upgrading it
+ args = ['sudo', 'rpm', '-ev', '%s-release' % project]
+ _run_and_log_error_if_fails(remote, args)
+
+ # Build the new -release package path
+ release_rpm = "{base}/noarch/{proj}-release-{release}.{dist_release}.noarch.rpm".format(
+ base=base_url,
+ proj=project,
+ release=RELEASE,
+ dist_release=distinfo['dist_release'],
+ )
+
+ # Upgrade the -release package
+ args = ['sudo', 'rpm', '-Uv', release_rpm]
+ _run_and_log_error_if_fails(remote, args)
+ _yum_fix_repo_priority(remote, project)
+
+ # Build a space-separated string consisting of $PKG-$VER for yum
+ pkgs_with_vers = ["%s-%s" % (pkg, version) for pkg in pkgs]
+
+ # Actually upgrade the project packages
+ # FIXME: This currently outputs nothing until the command is finished
+ # executing. That sucks; fix it.
+ args = ['sudo', 'yum', '-y', 'install']
+ args += pkgs_with_vers
+ _run_and_log_error_if_fails(remote, args)
+
+
@contextlib.contextmanager
def upgrade(ctx, config):
"""
branch = None
- proj_debs = {}
- proj_debs['ceph'] = [
- 'ceph',
- 'ceph-dbg',
- 'ceph-mds',
- 'ceph-mds-dbg',
- 'ceph-common',
- 'ceph-common-dbg',
- 'ceph-fuse',
- 'ceph-fuse-dbg',
- 'ceph-test',
- 'ceph-test-dbg',
- 'radosgw',
- 'radosgw-dbg',
- 'python-ceph',
- 'libcephfs1',
- 'libcephfs1-dbg',
- 'libcephfs-java',
- 'librados2',
- 'librados2-dbg',
- 'librbd1',
- 'librbd1-dbg',
- ]
-
project = config.get('project', 'ceph')
- debs = proj_debs.get(project, [])
-
+ # FIXME: extra_pkgs is not distro-agnostic
extra_pkgs = config.get('extra_packages', [])
log.info('extra packages: {packages}'.format(packages=extra_pkgs))
- debs += extra_pkgs
-
- log.info("Upgrading {proj} debian packages: {debs}".format(
- proj=project, debs=', '.join(debs)))
-
if config.get('all') is not None:
node = config.get('all')
if var == 'branch' or var == 'tag' or var == 'sha1':
branch = branch_val
for remote in ctx.cluster.remotes.iterkeys():
- _upgrade_packages(ctx, config, remote, debs, branch)
+ system_type = teuthology.get_system_type(remote)
+ assert system_type in ('deb', 'rpm')
+ pkgs = PACKAGES[project][system_type]
+ log.info("Upgrading {proj} {system_type} packages: {pkgs}".format(
+ proj=project, system_type=system_type, pkgs=', '.join(pkgs)))
+ # FIXME: again, make extra_pkgs distro-agnostic
+ pkgs += extra_pkgs
+ if system_type == 'deb':
+ _upgrade_deb_packages(ctx, config, remote, pkgs, branch)
+ elif system_type == 'rpm':
+ _upgrade_rpm_packages(ctx, config, remote, pkgs, branch)
else:
list_roles = []
for role in config.keys():
for var, branch_val in kkeys.iteritems():
if var == 'branch' or var == 'tag' or var == 'sha1':
branch = branch_val
- _upgrade_packages(ctx, config, remote, debs, branch)
+ system_type = teuthology.get_system_type(remote)
+ assert system_type in ('deb', 'rpm')
+ pkgs = PACKAGES[project][system_type]
+ _upgrade_deb_packages(ctx, config, remote, pkgs, branch)
list_roles.append(remote)
yield
flavor = 'gcov'
ctx.summary['flavor'] = flavor
-
+
with contextutil.nested(
lambda: install(ctx=ctx, config=dict(
branch=config.get('branch'),
projects / teuthology.git / commitdiff