From: Zack Cerza Date: Mon, 19 May 2014 23:21:09 +0000 (-0500) Subject: Refactoring X-Git-Tag: 1.1.0~1243 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=53a2c2f2d57b5cdabb5b7989c77f75f791471f47;p=teuthology.git Refactoring Create a new teuthology.provision module and move virtual machine functions there. Move hostname manipulation functions to teuthology.misc. Signed-off-by: Zack Cerza --- diff --git a/teuthology/lock.py b/teuthology/lock.py index 83513fb6c..1ffcf34c7 100644 --- a/teuthology/lock.py +++ b/teuthology/lock.py @@ -6,40 +6,18 @@ import urllib import yaml import re import collections -import tempfile import os import time import requests -import json import teuthology -from .config import config from . import lockstatus as ls from . import misc -from teuthology.misc import get_distro -from teuthology.misc import get_distro_version +from . import provision +from .config import config log = logging.getLogger(__name__) -hostname_expr = '(?P.*@)?(?P.*)\.front\.sepia\.ceph\.com' - - -def canonicalize_hostname(hostname, user='ubuntu'): - match = re.match(hostname_expr, hostname) - if match is None: - user_at = user + '@' if user else '' - hostname = '{user_at}{short}.front.sepia.ceph.com'.format( - user_at=user_at, - short=hostname) - return hostname - - -def decanonicalize_hostname(hostname): - match = re.match(hostname_expr, hostname) - if match: - hostname = match.groupdict()['shortname'] - return hostname - def lock_many(ctx, num, machinetype, user=None, description=None): machinetypes = misc.get_multi_machine_types(machinetype) @@ -63,7 +41,7 @@ def lock_many(ctx, num, machinetype, user=None, description=None): if machinetype == 'vps': ok_machs = {} for machine in machines: - if create_if_vm(ctx, machine): + if provision.create_if_vm(ctx, machine): ok_machs[machine] = machines[machine] else: log.error('Unable to create virtual machine: %s' % machine) @@ -106,7 +84,7 @@ def unlock_one(ctx, name, user=None): success = response.ok if success: log.debug('unlocked %s', name) - if not destroy_if_vm(ctx, name): + if not provision.destroy_if_vm(ctx, name): log.error('downburst destroy failed for %s', name) log.info('%s is not locked' % name) else: @@ -164,7 +142,7 @@ def main(ctx): ret = 0 user = ctx.owner - machines = [canonicalize_hostname(m) for m in ctx.machines] + machines = [misc.canonicalize_hostname(m) for m in ctx.machines] machines_to_update = [] if ctx.targets: @@ -284,7 +262,7 @@ def main(ctx): return ret else: machines_to_update.append(machine) - create_if_vm(ctx, machine) + provision.create_if_vm(ctx, machine) elif ctx.unlock: for machine in machines: if not unlock_one(ctx, machine, user): @@ -346,7 +324,7 @@ def updatekeys(ctx): misc.read_config(ctx) - machines = [canonicalize_hostname(m) for m in ctx.machines] + machines = [misc.canonicalize_hostname(m) for m in ctx.machines] if ctx.targets: try: @@ -430,113 +408,3 @@ def do_summary(ctx): print " --- ---" print "{cnt:12d} {up:3d}".format(cnt=total_count, up=total_up) - - -def _get_downburst_exec(): - """ - First check for downburst in the user's path. - Then check in ~/src, ~ubuntu/src, and ~teuthology/src. - Return '' if no executable downburst is found. - """ - if config.downburst: - return config.downburst - path = os.environ.get('PATH', None) - if path: - for p in os.environ.get('PATH', '').split(os.pathsep): - pth = os.path.join(p, 'downburst') - if os.access(pth, os.X_OK): - return pth - import pwd - little_old_me = pwd.getpwuid(os.getuid()).pw_name - for user in [little_old_me, 'ubuntu', 'teuthology']: - pth = "/home/%s/src/downburst/virtualenv/bin/downburst" % user - if os.access(pth, os.X_OK): - return pth - return '' - -# -# Use downburst to create a virtual machine -# - - -def create_if_vm(ctx, machine_name): - status_info = ls.get_status(ctx, machine_name) - phys_host = status_info['vpshost'] - if not phys_host: - return False - os_type = get_distro(ctx) - os_version = get_distro_version(ctx) - - createMe = decanonicalize_hostname(machine_name) - with tempfile.NamedTemporaryFile() as tmp: - if hasattr(ctx, 'config') and ctx.config is not None: - lcnfg = ctx.config.get('downburst', dict()) - else: - lcnfg = {} - distro = lcnfg.get('distro', os_type.lower()) - distroversion = lcnfg.get('distroversion', os_version) - - file_info = {} - file_info['disk-size'] = lcnfg.get('disk-size', '100G') - file_info['ram'] = lcnfg.get('ram', '1.9G') - file_info['cpus'] = lcnfg.get('cpus', 1) - file_info['networks'] = lcnfg.get('networks', - [{'source': 'front', 'mac': status_info['mac']}]) - file_info['distro'] = distro - file_info['distroversion'] = distroversion - file_info['additional-disks'] = lcnfg.get( - 'additional-disks', 3) - file_info['additional-disks-size'] = lcnfg.get( - 'additional-disks-size', '200G') - file_info['arch'] = lcnfg.get('arch', 'x86_64') - file_out = {'downburst': file_info} - yaml.safe_dump(file_out, tmp) - metadata = "--meta-data=%s" % tmp.name - dbrst = _get_downburst_exec() - if not dbrst: - log.error("No downburst executable found.") - return False - p = subprocess.Popen([dbrst, '-c', phys_host, - 'create', metadata, createMe], - stdout=subprocess.PIPE, stderr=subprocess.PIPE,) - owt, err = p.communicate() - if err: - log.info("Downburst completed on %s: %s" % - (machine_name, err)) - else: - log.info("%s created: %s" % (machine_name, owt)) - # If the guest already exists first destroy then re-create: - if 'exists' in err: - log.info("Guest files exist. Re-creating guest: %s" % - (machine_name)) - destroy_if_vm(ctx, machine_name) - create_if_vm(ctx, machine_name) - return True -# -# Use downburst to destroy a virtual machine -# - - -def destroy_if_vm(ctx, machine_name): - """ - Return False only on vm downburst failures. - """ - status_info = ls.get_status(machine_name) - phys_host = status_info['vm_host'] - if not phys_host: - return True - destroyMe = decanonicalize_hostname(machine_name) - dbrst = _get_downburst_exec() - if not dbrst: - log.error("No downburst executable found.") - return False - p = subprocess.Popen([dbrst, '-c', phys_host, - 'destroy', destroyMe], - stdout=subprocess.PIPE, stderr=subprocess.PIPE,) - owt, err = p.communicate() - if err: - log.error(err) - return False - else: - log.info("%s destroyed: %s" % (machine_name, owt)) - return True diff --git a/teuthology/misc.py b/teuthology/misc.py index b2194893e..6bddc86e3 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -34,6 +34,25 @@ is_vm = lambda x: x.startswith('vpm') or x.startswith('ubuntu@vpm') is_arm = lambda x: x.startswith('tala') or x.startswith( 'ubuntu@tala') or x.startswith('saya') or x.startswith('ubuntu@saya') +hostname_expr = '(?P.*@)?(?P.*)\.front\.sepia\.ceph\.com' + + +def canonicalize_hostname(hostname, user='ubuntu'): + match = re.match(hostname_expr, hostname) + if match is None: + user_at = user + '@' if user else '' + hostname = '{user_at}{short}.front.sepia.ceph.com'.format( + user_at=user_at, + short=hostname) + return hostname + + +def decanonicalize_hostname(hostname): + match = re.match(hostname_expr, hostname) + if match: + hostname = match.groupdict()['shortname'] + return hostname + def config_file(string): """ diff --git a/teuthology/provision.py b/teuthology/provision.py new file mode 100644 index 000000000..aba7fb38b --- /dev/null +++ b/teuthology/provision.py @@ -0,0 +1,121 @@ +import logging +import os +import subprocess +import tempfile +import yaml + +from .config import config +from .misc import decanonicalize_hostname +from .lockstatus import get_status +from .misc import get_distro +from .misc import get_distro_version + +log = logging.getLogger(__name__) + + +def _get_downburst_exec(): + """ + First check for downburst in the user's path. + Then check in ~/src, ~ubuntu/src, and ~teuthology/src. + Return '' if no executable downburst is found. + """ + if config.downburst: + return config.downburst + path = os.environ.get('PATH', None) + if path: + for p in os.environ.get('PATH', '').split(os.pathsep): + pth = os.path.join(p, 'downburst') + if os.access(pth, os.X_OK): + return pth + import pwd + little_old_me = pwd.getpwuid(os.getuid()).pw_name + for user in [little_old_me, 'ubuntu', 'teuthology']: + pth = "/home/%s/src/downburst/virtualenv/bin/downburst" % user + if os.access(pth, os.X_OK): + return pth + return '' + + +def create_if_vm(ctx, machine_name): + """ + Use downburst to create a virtual machine + """ + status_info = get_status(machine_name) + phys_host = status_info['vm_host'] + if not phys_host: + return False + os_type = get_distro(ctx) + os_version = get_distro_version(ctx) + + createMe = decanonicalize_hostname(machine_name) + with tempfile.NamedTemporaryFile() as tmp: + if hasattr(ctx, 'config') and ctx.config is not None: + lcnfg = ctx.config.get('downburst', dict()) + else: + lcnfg = {} + distro = lcnfg.get('distro', os_type.lower()) + distroversion = lcnfg.get('distroversion', os_version) + + file_info = {} + file_info['disk-size'] = lcnfg.get('disk-size', '100G') + file_info['ram'] = lcnfg.get('ram', '1.9G') + file_info['cpus'] = lcnfg.get('cpus', 1) + file_info['networks'] = lcnfg.get('networks', + [{'source': 'front', 'mac': status_info['mac']}]) + file_info['distro'] = distro + file_info['distroversion'] = distroversion + file_info['additional-disks'] = lcnfg.get( + 'additional-disks', 3) + file_info['additional-disks-size'] = lcnfg.get( + 'additional-disks-size', '200G') + file_info['arch'] = lcnfg.get('arch', 'x86_64') + file_out = {'downburst': file_info} + yaml.safe_dump(file_out, tmp) + metadata = "--meta-data=%s" % tmp.name + dbrst = _get_downburst_exec() + if not dbrst: + log.error("No downburst executable found.") + return False + p = subprocess.Popen([dbrst, '-c', phys_host, + 'create', metadata, createMe], + stdout=subprocess.PIPE, stderr=subprocess.PIPE,) + owt, err = p.communicate() + if err: + log.info("Downburst completed on %s: %s" % + (machine_name, err)) + else: + log.info("%s created: %s" % (machine_name, owt)) + # If the guest already exists first destroy then re-create: + if 'exists' in err: + log.info("Guest files exist. Re-creating guest: %s" % + (machine_name)) + destroy_if_vm(ctx, machine_name) + create_if_vm(ctx, machine_name) + return True + + +def destroy_if_vm(ctx, machine_name): + """ + Use downburst to destroy a virtual machine + + Return False only on vm downburst failures. + """ + status_info = get_status(machine_name) + phys_host = status_info['vm_host'] + if not phys_host: + return True + destroyMe = decanonicalize_hostname(machine_name) + dbrst = _get_downburst_exec() + if not dbrst: + log.error("No downburst executable found.") + return False + p = subprocess.Popen([dbrst, '-c', phys_host, + 'destroy', destroyMe], + stdout=subprocess.PIPE, stderr=subprocess.PIPE,) + owt, err = p.communicate() + if err: + log.error(err) + return False + else: + log.info("%s destroyed: %s" % (machine_name, owt)) + return True diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py index dd8d8f558..e584f1d7a 100644 --- a/teuthology/task/internal.py +++ b/teuthology/task/internal.py @@ -15,6 +15,7 @@ import subprocess from teuthology import lockstatus from teuthology import lock from teuthology import misc as teuthology +from teuthology import provision from teuthology.parallel import parallel from ..orchestra import cluster, remote, run @@ -123,8 +124,8 @@ def lock_machines(ctx, config): for guest in vmlist: if guest not in keyscan_out: log.info('recreating: ' + guest) - lock.destroy_if_vm(ctx, 'ubuntu@' + guest) - lock.create_if_vm(ctx, 'ubuntu@' + guest) + provision.destroy_if_vm(ctx, 'ubuntu@' + guest) + provision.create_if_vm(ctx, 'ubuntu@' + guest) if lock.update_keys(ctx, keyscan_out, current_locks): log.info("Error in virtual machine keys") newscandict = {} diff --git a/teuthology/test/test_lock.py b/teuthology/test/test_lock.py deleted file mode 100644 index 153083f9c..000000000 --- a/teuthology/test/test_lock.py +++ /dev/null @@ -1,24 +0,0 @@ -from .. import lock - - -class TestLock(object): - - def test_canonicalize_hostname(self): - host_base = 'box1' - result = lock.canonicalize_hostname(host_base) - assert result == 'ubuntu@box1.front.sepia.ceph.com' - - def test_decanonicalize_hostname(self): - host = 'ubuntu@box1.front.sepia.ceph.com' - result = lock.decanonicalize_hostname(host) - assert result == 'box1' - - def test_canonicalize_hostname_nouser(self): - host_base = 'box1' - result = lock.canonicalize_hostname(host_base, user=None) - assert result == 'box1.front.sepia.ceph.com' - - def test_decanonicalize_hostname_nouser(self): - host = 'box1.front.sepia.ceph.com' - result = lock.decanonicalize_hostname(host) - assert result == 'box1' diff --git a/teuthology/test/test_misc.py b/teuthology/test/test_misc.py index b2919e64e..e3e8c680d 100644 --- a/teuthology/test/test_misc.py +++ b/teuthology/test/test_misc.py @@ -50,3 +50,26 @@ def test_get_http_log_path(): path = misc.get_http_log_path(archive_dir) assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-master-testing-basic-vps/" + + +class TestHostnames(object): + + def test_canonicalize_hostname(self): + host_base = 'box1' + result = misc.canonicalize_hostname(host_base) + assert result == 'ubuntu@box1.front.sepia.ceph.com' + + def test_decanonicalize_hostname(self): + host = 'ubuntu@box1.front.sepia.ceph.com' + result = misc.decanonicalize_hostname(host) + assert result == 'box1' + + def test_canonicalize_hostname_nouser(self): + host_base = 'box1' + result = misc.canonicalize_hostname(host_base, user=None) + assert result == 'box1.front.sepia.ceph.com' + + def test_decanonicalize_hostname_nouser(self): + host = 'box1.front.sepia.ceph.com' + result = misc.decanonicalize_hostname(host) + assert result == 'box1'