From: Dan Mick Date: Fri, 14 Feb 2014 03:59:25 +0000 (-0800) Subject: Review: refactor packaging routines to common file X-Git-Tag: v0.94.10~27^2^2~364^2~376^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=006428c5cea27033f28afe12457e29c5cd61dbad;p=ceph.git Review: refactor packaging routines to common file Signed-off-by: Dan Mick --- diff --git a/teuthology/packaging.py b/teuthology/packaging.py new file mode 100644 index 000000000000..94fef24f17c7 --- /dev/null +++ b/teuthology/packaging.py @@ -0,0 +1,235 @@ +from cStringIO import StringIO +from .orchestra import run +import logging +import teuthology.misc as teuthology +import textwrap + +log = logging.getLogger(__name__) + +''' +Infer things about platform type with this map. +The key comes from processing lsb_release -ics or -irs (see _get_relmap). +''' +_RELEASE_MAP = { + 'Ubuntu precise': dict(flavor='deb', release='ubuntu', version='precise'), + 'Debian wheezy': dict(flavor='deb', release='debian', version='wheezy'), + 'CentOS 6.4': dict(flavor='rpm', release='centos', version='6.4'), + 'RedHatEnterpriseServer 6.4': dict(flavor='rpm', release='rhel', + version='6.4'), +} + +''' +Map 'generic' package name to 'flavor-specific' package name. +If entry is None, either the package isn't known here, or +it's known but should not be installed on remotes of this flavor +''' + +_PACKAGE_MAP = { + 'sqlite': {'deb': 'sqlite3', 'rpm': None} +} + +''' +Map 'generic' service name to 'flavor-specific' service name. +''' +_SERVICE_MAP = { + 'httpd': {'deb': 'apache2', 'rpm': 'httpd'} +} + + +def _get_relmap(rem): + """ + Internal worker to get the appropriate dict from RELEASE_MAP + """ + relmap = getattr(rem, 'relmap', None) + if relmap is not None: + return relmap + lsb_release_out = StringIO() + rem.run(args=['lsb_release', '-ics'], stdout=lsb_release_out) + release = lsb_release_out.getvalue().replace('\n', ' ').rstrip() + if release in _RELEASE_MAP: + rem.relmap = _RELEASE_MAP[release] + return rem.relmap + else: + lsb_release_out = StringIO() + rem.run(args=['lsb_release', '-irs'], stdout=lsb_release_out) + release = lsb_release_out.getvalue().replace('\n', ' ').rstrip() + if release in _RELEASE_MAP: + rem.relmap = _RELEASE_MAP[release] + return rem.relmap + raise RuntimeError('Can\'t get release info for {}'.format(rem)) + + +def get_package_name(pkg, rem): + """ + Find the remote-specific name of the generic 'pkg' + """ + flavor = _get_relmap(rem)['flavor'] + + try: + return _PACKAGE_MAP[pkg][flavor] + except KeyError: + return None + + +def get_service_name(service, rem): + """ + Find the remote-specific name of the generic 'service' + """ + flavor = _get_relmap(rem)['flavor'] + try: + return _SERVICE_MAP[service][flavor] + except KeyError: + return None + + +def install_repo(remote, reposerver, pkgdir, username=None, password=None): + """ + Install a package repo for reposerver on remote. + URL will be http if username and password are none, otherwise https. + pkgdir is the piece path between "reposerver" and "deb" or "rpm" + (say, 'packages', or 'packages-staging/my-branch', for example). + so: + http[s]://[:@]//{deb|rpm} + will be written to deb's apt inktank.list or rpm's inktank.repo + """ + + relmap = _get_relmap(remote) + log.info('Installing repo on %s', remote) + if username is None or password is None: + repo_uri = 'http://{reposerver}/{pkgdir}' + else: + repo_uri = 'https://{username}:{password}@{reposerver}/{pkgdir}' + + if relmap['flavor'] == 'deb': + contents = 'deb ' + repo_uri + '/deb {codename} main' + contents = contents.format(username=username, password=password, + reposerver=reposerver, pkgdir=pkgdir, + codename=relmap['version'],) + teuthology.sudo_write_file(remote, + '/etc/apt/sources.list.d/inktank.list', + contents) + remote.run(args=['sudo', + 'apt-get', + 'install', + 'apt-transport-https', + '-y']) + result = remote.run(args=['sudo', 'apt-get', 'update', '-y'], + stdout=StringIO()) + return result + + elif relmap['flavor'] == 'rpm': + baseurl = repo_uri + '/rpm/{release}{version}' + contents = textwrap.dedent(''' + [inktank] + name=Inktank Storage, Inc. + baseurl={baseurl} + gpgcheck=1 + enabled=1 + '''.format(baseurl=baseurl)) + contents = contents.format(username=username, + password=password, + pkgdir=pkgdir, + release=relmap['release'], + version=relmap['version']) + teuthology.sudo_write_file(remote, + '/etc/yum.repos.d/inktank.repo', + contents) + return remote.run(args=['sudo', 'yum', 'makecache']) + + else: + return False + + +def remove_repo(remote): + log.info('Removing repo on %s', remote) + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + teuthology.delete_file(remote, '/etc/apt/sources.list.d/inktank.list', + sudo=True, force=True) + result = remote.run(args=['sudo', 'apt-get', 'update', '-y'], + stdout=StringIO()) + return result + + elif flavor == 'rpm': + teuthology.delete_file(remote, '/etc/yum.repos.d/inktank.repo', + sudo=True, force=True) + return remote.run(args=['sudo', 'yum', 'makecache']) + + else: + return False + + +def install_repokey(remote, keyurl): + """ + Install a repo key from keyurl on remote. + Installing keys is assumed to be idempotent. + Example keyurl: 'http://download.inktank.com/keys/release.asc' + """ + log.info('Installing repo key on %s', remote) + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + return remote.run(args=['wget', + '-q', + '-O-', + keyurl, + run.Raw('|'), + 'sudo', + 'apt-key', + 'add', + '-']) + elif flavor == 'rpm': + return remote.run(args=['sudo', 'rpm', '--import', keyurl]) + else: + return False + + +def install_package(package, remote): + """ + Install 'package' on 'remote' + Assumes repo has already been set up (perhaps with install_repo) + """ + log.info('Installing package %s on %s', package, remote) + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + pkgcmd = ['DEBIAN_FRONTEND=noninteractive', + 'sudo', + '-E', + 'apt-get', + '-y', + 'install', + '{package}'.format(package=package)] + elif flavor == 'rpm': + pkgcmd = ['sudo', + 'yum', + '-y', + 'install', + '{package}'.format(package=package)] + else: + log.error('install_package: bad flavor ' + flavor + '\n') + return False + return remote.run(args=pkgcmd) + + +def remove_package(package, remote): + """ + Remove package from remote + """ + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + pkgcmd = ['DEBIAN_FRONTEND=noninteractive', + 'sudo', + '-E', + 'apt-get', + '-y', + 'purge', + '{package}'.format(package=package)] + elif flavor == 'rpm': + pkgcmd = ['sudo', + 'yum', + '-y', + 'erase', + '{package}'.format(package=package)] + else: + log.error('remove_package: bad flavor ' + flavor + '\n') + return False + return remote.run(args=pkgcmd) diff --git a/teuthology/task/calamari.py b/teuthology/task/calamari.py index 221004033cdb..1e49c185d564 100644 --- a/teuthology/task/calamari.py +++ b/teuthology/task/calamari.py @@ -22,6 +22,7 @@ import logging import os import subprocess import teuthology.misc as teuthology +import teuthology.packaging as pkg import textwrap import time from ..orchestra import run @@ -57,7 +58,7 @@ def _disable_default_nginx(remote): service nginx restart service {service} restart ''') - service = _http_service_name(remote) + service = pkg.get_service_name('httpd', remote) script = script.format(service=service) teuthology.sudo_write_file(remote, '/tmp/disable.nginx', script) return remote.run(args=['sudo', 'bash', '/tmp/disable.nginx'], @@ -82,186 +83,6 @@ def _setup_calamari_cluster(remote, restapi_remote): stdout=StringIO()) -RELEASE_MAP = { - 'Ubuntu precise': dict(flavor='deb', release='ubuntu', version='precise'), - 'Debian wheezy': dict(flavor='deb', release='debian', version='wheezy'), - 'CentOS 6.4': dict(flavor='rpm', release='centos', version='6.4'), - 'RedHatEnterpriseServer 6.4': dict(flavor='rpm', release='rhel', - version='6.4'), -} - - -def _get_relmap(rem): - relmap = getattr(rem, 'relmap', None) - if relmap is not None: - return relmap - lsb_release_out = StringIO() - rem.run(args=['lsb_release', '-ics'], stdout=lsb_release_out) - release = lsb_release_out.getvalue().replace('\n', ' ').rstrip() - if release in RELEASE_MAP: - rem.relmap = RELEASE_MAP[release] - return rem.relmap - else: - lsb_release_out = StringIO() - rem.run(args=['lsb_release', '-irs'], stdout=lsb_release_out) - release = lsb_release_out.getvalue().replace('\n', ' ').rstrip() - if release in RELEASE_MAP: - rem.relmap = RELEASE_MAP[release] - return rem.relmap - raise RuntimeError('Can\'t get release info for {}'.format(rem)) - - -def _sqlite_package_name(rem): - name = 'sqlite3' if _get_relmap(rem)['flavor'] == 'deb' else None - return name - - -def _http_service_name(rem): - name = 'httpd' if _get_relmap(rem)['flavor'] == 'rpm' else 'apache2' - return name - - -def _install_repo(remote, pkgdir, username, password): - # installing repo is assumed to be idempotent - - relmap = _get_relmap(remote) - log.info('Installing repo on %s', remote) - if relmap['flavor'] == 'deb': - contents = 'deb https://{username}:{password}@download.inktank.com/' \ - '{pkgdir}/deb {codename} main' - contents = contents.format(username=username, password=password, - pkgdir=pkgdir, codename=relmap['version'],) - teuthology.sudo_write_file(remote, - '/etc/apt/sources.list.d/inktank.list', - contents) - remote.run(args=['sudo', - 'apt-get', - 'install', - 'apt-transport-https', - '-y']) - result = remote.run(args=['sudo', 'apt-get', 'update', '-y'], - stdout=StringIO()) - return result - - elif relmap['flavor'] == 'rpm': - baseurl = 'https://{username}:{password}@download.inktank.com/' \ - '{pkgdir}/rpm/{release}{version}' - contents = textwrap.dedent(''' - [inktank] - name=Inktank Storage, Inc. - baseurl={baseurl} - gpgcheck=1 - enabled=1 - '''.format(baseurl=baseurl)) - contents = contents.format(username=username, - password=password, - pkgdir=pkgdir, - release=relmap['release'], - version=relmap['version']) - teuthology.sudo_write_file(remote, - '/etc/yum.repos.d/inktank.repo', - contents) - return remote.run(args=['sudo', 'yum', 'makecache']) - - else: - return False - - -def _remove_repo(remote): - log.info('Removing repo on %s', remote) - flavor = _get_relmap(remote)['flavor'] - if flavor == 'deb': - teuthology.delete_file(remote, '/etc/apt/sources.list.d/inktank.list', - sudo=True, force=True) - result = remote.run(args=['sudo', 'apt-get', 'update', '-y'], - stdout=StringIO()) - return result - - elif flavor == 'rpm': - teuthology.delete_file(remote, '/etc/yum.repos.d/inktank.repo', - sudo=True, force=True) - return remote.run(args=['sudo', 'yum', 'makecache']) - - else: - return False - - -def _install_repokey(remote): - # installing keys is assumed to be idempotent - log.info('Installing repo key on %s', remote) - flavor = _get_relmap(remote)['flavor'] - if flavor == 'deb': - return remote.run(args=['wget', - '-q', - '-O-', - 'http://download.inktank.com/keys/release.asc', - run.Raw('|'), - 'sudo', - 'apt-key', - 'add', - '-']) - elif flavor == 'rpm': - args = ['sudo', 'rpm', '--import', - 'http://download.inktank.com/keys/release.asc'] - return remote.run(args=args) - else: - return False - - -def _install_package(package, remote): - """ - package: name - remote: Remote() to install on - release: deb only, 'precise' or 'wheezy' - pkgdir: may or may not include a branch name, so, say, either - packages or packages-staging/master - """ - log.info('Installing package %s on %s', package, remote) - flavor = _get_relmap(remote)['flavor'] - if flavor == 'deb': - pkgcmd = ['DEBIAN_FRONTEND=noninteractive', - 'sudo', - '-E', - 'apt-get', - '-y', - 'install', - '{package}'.format(package=package)] - elif flavor == 'rpm': - pkgcmd = ['sudo', - 'yum', - '-y', - 'install', - '{package}'.format(package=package)] - else: - log.error('_install_package: bad flavor ' + flavor + '\n') - return False - return remote.run(args=pkgcmd) - - -def _remove_package(package, remote): - """ - remove package from remote - """ - flavor = _get_relmap(remote)['flavor'] - if flavor == 'deb': - pkgcmd = ['DEBIAN_FRONTEND=noninteractive', - 'sudo', - '-E', - 'apt-get', - '-y', - 'purge', - '{package}'.format(package=package)] - elif flavor == 'rpm': - pkgcmd = ['sudo', - 'yum', - '-y', - 'erase', - '{package}'.format(package=package)] - else: - log.error('_remove_package: bad flavor ' + flavor + '\n') - return False - return remote.run(args=pkgcmd) - """ Tasks """ @@ -292,7 +113,7 @@ def agent(ctx, config): try: for rem in remotes: log.info('Installing calamari-agent on %s', rem) - _install_package('calamari-agent', rem) + pkg.install_package('calamari-agent', rem) server_role = config.get('server') if not server_role: raise RuntimeError('must supply \'server\' config key') @@ -307,7 +128,7 @@ def agent(ctx, config): yield finally: for rem in remotes: - _remove_package('calamari-agent', rem) + pkg.remove_package('calamari-agent', rem) @contextlib.contextmanager @@ -347,13 +168,15 @@ def reposetup(ctx, config): try: for rem in remotes: log.info(rem) - _install_repokey(rem) - _install_repo(rem, pkgdir, username, password) + keypath = 'http://download.inktank.com/keys/release.asc' + pkg.install_repokey(rem, keypath) + pkg.install_repo(rem, 'download.inktank.com', pkgdir, + username, password) yield finally: for rem in remotes: - _remove_repo(rem) + pkg.remove_repo(rem) @contextlib.contextmanager @@ -377,12 +200,12 @@ def restapi(ctx, config): try: for rem in remotes: log.info(rem) - _install_package('calamari-restapi', rem) + pkg.install_package('calamari-restapi', rem) yield finally: for rem in remotes: - _remove_package('calamari-restapi', rem) + pkg.remove_package('calamari-restapi', rem) @contextlib.contextmanager @@ -415,12 +238,12 @@ def server(ctx, config): try: # sqlite3 command is required; on some platforms it's already # there and not removable (required for, say yum) - sqlite_package = _sqlite_package_name(remote) - if sqlite_package and not _install_package(sqlite_package, remote): + sqlite_package = pkg.get_package_name('sqlite', remote) + if sqlite_package and not pkg.install_package(sqlite_package, remote): raise RuntimeError('{} install failed'.format(sqlite_package)) - if not _install_package('calamari-server', remote) or \ - not _install_package('calamari-clients', remote) or \ + if not pkg.install_package('calamari-server', remote) or \ + not pkg.install_package('calamari-clients', remote) or \ not _disable_default_nginx(remote) or \ not _setup_calamari_cluster(remote, restapi_remote): raise RuntimeError('Server installation failure') @@ -428,10 +251,10 @@ def server(ctx, config): log.info('client/server setup complete') yield finally: - _remove_package('calamari-server', remote) - _remove_package('calamari-clients', remote) + pkg.remove_package('calamari-server', remote) + pkg.remove_package('calamari-clients', remote) if sqlite_package: - _remove_package(sqlite_package, remote) + pkg.remove_package(sqlite_package, remote) def test(ctx, config):