From: Zack Cerza Date: Fri, 13 Feb 2015 17:41:36 +0000 (-0700) Subject: Move downburst code into new Downburst class X-Git-Tag: 1.1.0~974^2~3 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=3403fcbf5d80f5a30db338ac8e73eabd0f36f387;p=teuthology.git Move downburst code into new Downburst class Signed-off-by: Zack Cerza --- diff --git a/teuthology/provision.py b/teuthology/provision.py index 42788ae44e..abcea824f0 100644 --- a/teuthology/provision.py +++ b/teuthology/provision.py @@ -5,6 +5,7 @@ import tempfile import yaml from .config import config +from .contextutil import safe_while from .misc import decanonicalize_hostname, get_distro, get_distro_version from .lockstatus import get_status @@ -12,7 +13,7 @@ from .lockstatus import get_status log = logging.getLogger(__name__) -def _get_downburst_exec(): +def downburst_executable(): """ First check for downburst in the user's path. Then check in ~/src, ~ubuntu/src, and ~teuthology/src. @@ -29,76 +30,167 @@ def _get_downburst_exec(): import pwd little_old_me = pwd.getpwuid(os.getuid()).pw_name for user in [little_old_me, 'ubuntu', 'teuthology']: - pth = os.path.expanduser("~%s/src/downburst/virtualenv/bin/downburst" - % user) + pth = os.path.expanduser( + "~%s/src/downburst/virtualenv/bin/downburst" % user) if os.access(pth, os.X_OK): return pth return '' -def create_if_vm(ctx, machine_name): +class Downburst(object): """ - Use downburst to create a virtual machine + A class that provides methods for creating and destroying virtual machine + instances using downburst: https://github.com/ceph/downburst """ - status_info = get_status(machine_name) - if not status_info.get('is_vm', False): - return False - phys_host = decanonicalize_hostname(status_info['vm_host']['name']) - os_type = get_distro(ctx) - os_version = get_distro_version(ctx) + def __init__(self, name, os_type, os_version, status=None): + self.name = name + self.os_type = os_type + self.os_version = os_version + self.status = status or get_status(self.name) + self.config_path = None + self.host = decanonicalize_hostname(self.status['vm_host']['name']) + self.executable = downburst_executable() + + def create(self): + """ + Launch a virtual machine instance. + + If creation fails because an instance with the specified name is + already running, first destroy it, then try again. This process will + repeat two more times, waiting 60s between tries, before giving up. + """ + if not self.executable: + log.error("No downburst executable found.") + return False + self.build_config() + success = None + with safe_while(sleep=60, tries=3, + action="downburst create") as proceed: + while proceed(): + (returncode, stdout, stderr) = self._run_create() + if returncode == 0: + log.info("Downburst created %s: %s" % (self.name, + stdout.strip())) + success = True + break + elif stderr: + # If the guest already exists first destroy then re-create: + if 'exists' in stderr: + success = False + log.info("Guest files exist. Re-creating guest: %s" % + (self.name)) + self.destroy() + else: + success = False + log.info("Downburst failed on %s: %s" % ( + self.name, stderr.strip())) + break + return success - createMe = decanonicalize_hostname(machine_name) - with tempfile.NamedTemporaryFile() as tmp: - has_config = hasattr(ctx, 'config') and ctx.config is not None - if has_config and 'downburst' in ctx.config: - log.warning( - 'Usage of a custom downburst config has been deprecated.' - ) + def _run_create(self): + """ + Used by create(), this method is what actually calls downburst when + creating a virtual machine instance. + """ + if not self.config_path: + raise ValueError("I need a config_path!") + shortname = decanonicalize_hostname(self.name) + args = [self.executable, '-c', self.host, 'create', + '--meta-data=%s' % self.config_path, shortname, + ] log.info("Provisioning a {distro} {distroversion} vps".format( - distro=os_type, - distroversion=os_version + distro=self.os_type, + distroversion=self.os_version )) + proc = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = proc.communicate() + return (proc.returncode, out, err) + + def destroy(self): + """ + Destroy (shutdown and delete) a virtual machine instance. + """ + executable = self.executable + if not executable: + log.error("No downburst executable found.") + return False + shortname = decanonicalize_hostname(self.name) + args = [executable, '-c', self.host, 'destroy', shortname] + proc = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.PIPE,) + out, err = proc.communicate() + if err: + log.error("Error destroying {machine}: {msg}".format( + machine=self.name, msg=err)) + return False + elif proc.returncode == 0: + out_str = ': %s' % out if out else '' + log.info("Destroyed %s%s" % (self.name, out_str)) + return True + else: + log.error("I don't know if the destroy of {node} succeded!".format( + node=self.name)) + return False + + def build_config(self): + """ + Assemble a configuration to pass to downburst, and write it to a file. + """ + config_fd = tempfile.NamedTemporaryFile(delete=False) file_info = { 'disk-size': '100G', 'ram': '1.9G', 'cpus': 1, 'networks': [ - {'source': 'front', 'mac': status_info['mac_address']}], - 'distro': os_type.lower(), - 'distroversion': os_version, + {'source': 'front', 'mac': self.status['mac_address']}], + 'distro': self.os_type.lower(), + 'distroversion': self.os_version, 'additional-disks': 3, 'additional-disks-size': '200G', 'arch': 'x86_64', } - fqdn = machine_name.split('@')[1] + fqdn = self.name.split('@')[1] file_out = {'downburst': file_info, 'local-hostname': fqdn} - yaml.safe_dump(file_out, tmp) - metadata = "--meta-data=%s" % tmp.name - dbrst = _get_downburst_exec() - if not dbrst: - log.error("No downburst executable found.") - return False - p = subprocess.Popen([dbrst, '-c', phys_host, - 'create', metadata, createMe], - stdout=subprocess.PIPE, stderr=subprocess.PIPE,) - owt, err = p.communicate() - if p.returncode == 0: - log.info("Downburst created %s: %s" % (machine_name, owt.strip())) - elif err: - # If the guest already exists first destroy then re-create: - # FIXME there is a recursion-depth issue here. - if 'exists' in err: - log.info("Guest files exist. Re-creating guest: %s" % - (machine_name)) - destroy_if_vm(ctx, machine_name) - create_if_vm(ctx, machine_name) - else: - log.info("Downburst failed on %s: %s" % (machine_name, - err.strip())) - return False - return True + yaml.safe_dump(file_out, config_fd) + self.config_path = config_fd.name + return True + + def remove_config(self): + """ + Remove the downburst configuration file created by build_config() + """ + if self.config_path and os.path.exists(self.config_path): + os.remove(self.config_path) + self.config_path = None + return True + return False + + def __del__(self): + self.remove_config() + + +def create_if_vm(ctx, machine_name): + """ + Use downburst to create a virtual machine + """ + status_info = get_status(machine_name) + if not status_info.get('is_vm', False): + return False + os_type = get_distro(ctx) + os_version = get_distro_version(ctx) + + has_config = hasattr(ctx, 'config') and ctx.config is not None + if has_config and 'downburst' in ctx.config: + log.warning( + 'Usage of a custom downburst config has been deprecated.' + ) + + dbrst = Downburst(name=machine_name, os_type=os_type, + os_version=os_version, status=status_info) + return dbrst.create() def destroy_if_vm(ctx, machine_name, user=None, description=None): @@ -108,6 +200,8 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None): Return False only on vm downburst failures. """ status_info = get_status(machine_name) + os_type = get_distro(ctx) + os_version = get_distro_version(ctx) if not status_info or not status_info.get('is_vm', False): return True if user is not None and user != status_info['locked_by']: @@ -118,20 +212,6 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None): log.error("Tried to destroy {node} with description {desc_arg} but it is locked with description {desc_lock}".format( node=machine_name, desc_arg=description, desc_lock=status_info['description'])) return False - phys_host = decanonicalize_hostname(status_info['vm_host']['name']) - destroyMe = decanonicalize_hostname(machine_name) - dbrst = _get_downburst_exec() - if not dbrst: - log.error("No downburst executable found.") - return False - p = subprocess.Popen([dbrst, '-c', phys_host, - 'destroy', destroyMe], - stdout=subprocess.PIPE, stderr=subprocess.PIPE,) - owt, err = p.communicate() - if err: - log.error("Error destroying {machine}: {msg}".format( - machine=machine_name, msg=err)) - return False - else: - log.info("%s destroyed: %s" % (machine_name, owt)) - return True + dbrst = Downburst(name=machine_name, os_type=os_type, + os_version=os_version, status=status_info) + return dbrst.destroy()