From 53776452e8c38783a056d22509462885b619c0eb Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 5 Jan 2017 17:04:11 -0700 Subject: [PATCH] Make teuthology.lock a subpackage Signed-off-by: Zack Cerza --- docs/_static/create_nodes.py | 3 +- scripts/lock.py | 3 +- scripts/update_inventory.py | 3 +- scripts/updatekeys.py | 3 +- teuthology/lock.py | 778 ------------------ teuthology/lock/__init__.py | 1 + teuthology/lock/cli.py | 266 ++++++ teuthology/lock/keys.py | 27 + teuthology/lock/ops.py | 237 ++++++ teuthology/lock/query.py | 141 ++++ teuthology/lock/test/__init__.py | 0 teuthology/{ => lock}/test/test_lock.py | 5 +- teuthology/lock/util.py | 121 +++ teuthology/misc.py | 6 +- teuthology/nuke/__init__.py | 27 +- .../openstack/test/openstack-integration.py | 21 +- teuthology/orchestra/console.py | 7 +- teuthology/orchestra/remote.py | 12 +- teuthology/provision/__init__.py | 28 +- teuthology/provision/downburst.py | 43 +- teuthology/provision/test/test_downburst.py | 16 +- teuthology/suite/test/test_util.py | 16 +- teuthology/suite/util.py | 5 +- teuthology/task/internal/__init__.py | 4 +- teuthology/task/internal/check_lock.py | 5 +- teuthology/task/internal/lock_machines.py | 23 +- .../test_vps_os_vers_parameter_checking.py | 54 +- 27 files changed, 960 insertions(+), 895 deletions(-) delete mode 100644 teuthology/lock.py create mode 100644 teuthology/lock/__init__.py create mode 100644 teuthology/lock/cli.py create mode 100644 teuthology/lock/keys.py create mode 100644 teuthology/lock/ops.py create mode 100644 teuthology/lock/query.py create mode 100644 teuthology/lock/test/__init__.py rename teuthology/{ => lock}/test/test_lock.py (57%) create mode 100644 teuthology/lock/util.py diff --git a/docs/_static/create_nodes.py b/docs/_static/create_nodes.py index 184dfe7c2f..3645b613ca 100755 --- a/docs/_static/create_nodes.py +++ b/docs/_static/create_nodes.py @@ -8,7 +8,8 @@ import logging import sys from teuthology.orchestra.remote import Remote -from teuthology.lock import update_inventory +from teuthology.lock.ops import update_inventory + paddles_url = 'http://paddles.example.com/nodes/' machine_type = 'typica' diff --git a/scripts/lock.py b/scripts/lock.py index 6d4b9e168b..76b81d95ea 100644 --- a/scripts/lock.py +++ b/scripts/lock.py @@ -3,6 +3,7 @@ import textwrap import sys import teuthology.lock +import teuthology.lock.cli def _positive_int(string): @@ -14,7 +15,7 @@ def _positive_int(string): def main(): - sys.exit(teuthology.lock.main(parse_args(sys.argv[1:]))) + sys.exit(teuthology.lock.cli.main(parse_args(sys.argv[1:]))) def parse_args(argv): diff --git a/scripts/update_inventory.py b/scripts/update_inventory.py index e281231d6a..3185e44149 100644 --- a/scripts/update_inventory.py +++ b/scripts/update_inventory.py @@ -2,6 +2,7 @@ import docopt import teuthology import teuthology.lock +import teuthology.lock.ops import teuthology.orchestra.remote import logging @@ -29,5 +30,5 @@ def main(): for rem_name in remotes: remote = teuthology.orchestra.remote.Remote(rem_name) inventory_info = remote.inventory_info - teuthology.lock.update_inventory(inventory_info) + teuthology.lock.ops.update_inventory(inventory_info) diff --git a/scripts/updatekeys.py b/scripts/updatekeys.py index 524c9c07e6..eb4ce25207 100644 --- a/scripts/updatekeys.py +++ b/scripts/updatekeys.py @@ -2,6 +2,7 @@ import docopt import sys import teuthology.lock +import teuthology.lock.cli doc = """ usage: teuthology-updatekeys -h @@ -24,5 +25,5 @@ optional arguments: def main(): args = docopt.docopt(doc) - status = teuthology.lock.updatekeys(args) + status = teuthology.lock.cli.updatekeys(args) sys.exit(status) diff --git a/teuthology/lock.py b/teuthology/lock.py deleted file mode 100644 index bc32d75d7d..0000000000 --- a/teuthology/lock.py +++ /dev/null @@ -1,778 +0,0 @@ -import argparse -import datetime -import json -import logging -import subprocess -import yaml -import re -import collections -import os -import requests -import urllib - -import teuthology -from . import misc -from . import provision -from .config import config -from .config import set_config_attr -from .contextutil import safe_while - -log = logging.getLogger(__name__) - - -def get_status(name): - name = misc.canonicalize_hostname(name, user=None) - uri = os.path.join(config.lock_server, 'nodes', name, '') - response = requests.get(uri) - success = response.ok - if success: - return response.json() - log.warning( - "Failed to query lock server for status of {name}".format(name=name)) - return None - - -def is_vm(name=None, status=None): - if status is None: - if name is None: - raise ValueError("Must provide either name or status, or both") - name = misc.canonicalize_hostname(name) - status = get_status(name) - return status.get('is_vm', False) - - -def get_distro_from_downburst(): - """ - Return a table of valid distros. - - If downburst is in path use it. If either downburst is unavailable, - or if downburst is unable to produce a json list, then use a default - table. - """ - default_table = {u'rhel_minimal': [u'6.4', u'6.5'], - u'fedora': [u'17', u'18', u'19', u'20', u'22'], - u'centos': [u'6.3', u'6.4', u'6.5', u'7.0', - u'7.2'], - u'centos_minimal': [u'6.4', u'6.5'], - u'ubuntu': [u'8.04(hardy)', u'9.10(karmic)', - u'10.04(lucid)', u'10.10(maverick)', - u'11.04(natty)', u'11.10(oneiric)', - u'12.04(precise)', u'12.10(quantal)', - u'13.04(raring)', u'13.10(saucy)', - u'14.04(trusty)', u'utopic(utopic)', - u'16.04(xenial)'], - u'sles': [u'11-sp2'], - u'debian': [u'6.0', u'7.0', u'8.0']} - executable_cmd = provision.downburst.downburst_executable() - if not executable_cmd: - log.warn("Downburst not found!") - log.info('Using default values for supported os_type/os_version') - return default_table - try: - output = subprocess.check_output([executable_cmd, 'list-json']) - downburst_data = json.loads(output) - return downburst_data - except (subprocess.CalledProcessError, OSError): - log.exception("Error calling downburst!") - log.info('Using default values for supported os_type/os_version') - return default_table - - -def vps_version_or_type_valid(machine_type, os_type, os_version): - """ - Check os-type and os-version parameters when locking a vps. - Os-type will always be set (defaults to ubuntu). - - In the case where downburst does not handle list-json (an older version - of downburst, for instance), a message is printed and this checking - is skipped (so that this code should behave as it did before this - check was added). - """ - if not machine_type == 'vps': - return True - if os_type is None or os_version is None: - # we'll use the defaults provided by provision.create_if_vm - # later on during provisioning - return True - valid_os_and_version = get_distro_from_downburst() - if os_type not in valid_os_and_version: - log.error("os-type '%s' is invalid. Try one of: %s", - os_type, - ', '.join(valid_os_and_version.keys())) - return False - if not validate_distro_version(os_version, - valid_os_and_version[os_type]): - log.error( - "os-version '%s' is invalid for os-type '%s'. Try one of: %s", - os_version, - os_type, - ', '.join(valid_os_and_version[os_type])) - return False - return True - - -def validate_distro_version(version, supported_versions): - """ - Return True if the version is valid. For Ubuntu, possible - supported version values are of the form '12.04 (precise)' where - either the number of the version name is acceptable. - """ - if version in supported_versions: - return True - for parts in supported_versions: - part = parts.split('(') - if len(part) == 2: - if version == part[0]: - return True - if version == part[1][0:len(part[1])-1]: - return True - - -def get_statuses(machines): - if machines: - statuses = [] - for machine in machines: - machine = misc.canonicalize_hostname(machine) - status = get_status(machine) - if status: - statuses.append(status) - else: - log.error("Lockserver doesn't know about machine: %s" % - machine) - else: - statuses = list_locks() - return statuses - - -def json_matching_statuses(json_file_or_str, statuses): - """ - Filter statuses by json dict in file or fragment; return list of - matching statuses. json_file_or_str must be a file containing - json or json in a string. - """ - try: - open(json_file_or_str, 'r') - except IOError: - query = json.loads(json_file_or_str) - else: - query = json.load(json_file_or_str) - - if not isinstance(query, dict): - raise RuntimeError('--json-query must be a dict') - - return_statuses = list() - for status in statuses: - for k, v in query.iteritems(): - if not misc.is_in_dict(k, v, status): - break - else: - return_statuses.append(status) - - return return_statuses - - -def winnow(statuses, arg, status_key, func=None): - """ - Call with a list of statuses, and the ctx. - 'arg' that you may want to filter by. - If arg is not None, filter statuses by either: - - 1) func=None: filter by status[status_key] == arg - remove any status that fails - - 2) func=: remove any - status for which func returns False - - Return the possibly-smaller set of statuses. - """ - - if arg is not None: - if func: - statuses = [_status for _status in statuses - if func(_status)] - else: - statuses = [_status for _status in statuses - if _status[status_key] == arg] - - return statuses - - -def main(ctx): - if ctx.verbose: - teuthology.log.setLevel(logging.DEBUG) - - set_config_attr(ctx) - - ret = 0 - user = ctx.owner - machines = [misc.canonicalize_hostname(m, user=False) - for m in ctx.machines] - machines_to_update = [] - - if ctx.targets: - try: - with file(ctx.targets) as f: - g = yaml.safe_load_all(f) - for new in g: - if 'targets' in new: - for t in new['targets'].iterkeys(): - machines.append(t) - except IOError as e: - raise argparse.ArgumentTypeError(str(e)) - - if ctx.f: - assert ctx.lock or ctx.unlock, \ - '-f is only supported by --lock and --unlock' - if machines: - assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \ - or ctx.update or ctx.brief, \ - 'machines cannot be specified with that operation' - else: - if ctx.lock: - log.error("--lock requires specific machines passed as arguments") - else: - # This condition might never be hit, but it's not clear. - assert ctx.num_to_lock or ctx.list or ctx.list_targets or \ - ctx.summary or ctx.brief, \ - 'machines must be specified for that operation' - if ctx.all: - assert ctx.list or ctx.list_targets or ctx.brief, \ - '--all can only be used with --list, --list-targets, and --brief' - assert ctx.owner is None, \ - '--all and --owner are mutually exclusive' - assert not machines, \ - '--all and listing specific machines are incompatible' - if ctx.num_to_lock: - assert ctx.machine_type, \ - 'must specify machine type to lock' - - if ctx.brief or ctx.list or ctx.list_targets: - assert ctx.desc is None, '--desc does nothing with --list/--brief' - - # we may need to update host keys for vms. Don't do it for - # every vm; however, update any vms included in the list given - # to the CLI (machines), or any owned by the specified owner or - # invoking user if no machines are specified. - vmachines = [] - statuses = get_statuses(machines) - owner = ctx.owner or misc.get_user() - for machine in statuses: - if is_vm(status=machine) and machine['locked'] and \ - (machines or machine['locked_by'] == owner): - vmachines.append(machine['name']) - if vmachines: - log.info("updating host keys for %s", ' '.join(sorted(vmachines))) - do_update_keys(vmachines) - # get statuses again to refresh any updated keys - statuses = get_statuses(machines) - if statuses: - statuses = winnow(statuses, ctx.machine_type, 'machine_type') - if not machines and ctx.owner is None and not ctx.all: - ctx.owner = misc.get_user() - statuses = winnow(statuses, ctx.owner, 'locked_by') - statuses = winnow(statuses, ctx.status, 'up', - lambda s: s['up'] == (ctx.status == 'up')) - statuses = winnow(statuses, ctx.locked, 'locked', - lambda s: s['locked'] == (ctx.locked == 'true')) - statuses = winnow(statuses, ctx.desc, 'description') - statuses = winnow(statuses, ctx.desc_pattern, 'description', - lambda s: s['description'] and \ - ctx.desc_pattern in s['description']) - if ctx.json_query: - statuses = json_matching_statuses(ctx.json_query, statuses) - statuses = winnow(statuses, ctx.os_type, 'os_type') - statuses = winnow(statuses, ctx.os_version, 'os_version') - - # When listing, only show the vm_host's name, not every detail - for s in statuses: - if not is_vm(status=s): - continue - # with an OpenStack API, there is no host for a VM - if s['vm_host'] is None: - continue - vm_host_name = s.get('vm_host', dict())['name'] - if vm_host_name: - s['vm_host'] = vm_host_name - if ctx.list: - print json.dumps(statuses, indent=4) - - elif ctx.brief: - for s in sorted(statuses, key=lambda s: s.get('name')): - locked = "un" if s['locked'] == 0 else " " - mo = re.match('\w+@(\w+?)\..*', s['name']) - host = mo.group(1) if mo else s['name'] - print '{host} {locked}locked {owner} "{desc}"'.format( - locked=locked, host=host, - owner=s['locked_by'], desc=s['description']) - - else: - frag = {'targets': {}} - for f in statuses: - frag['targets'][f['name']] = f['ssh_pub_key'] - print yaml.safe_dump(frag, default_flow_style=False) - else: - log.error('error retrieving lock statuses') - ret = 1 - - elif ctx.summary: - do_summary(ctx) - return 0 - - elif ctx.lock: - if not vps_version_or_type_valid(ctx.machine_type, ctx.os_type, - ctx.os_version): - log.error('Invalid os-type or version detected -- lock failed') - return 1 - for machine in machines: - if not lock_one(machine, user, ctx.desc): - ret = 1 - if not ctx.f: - return ret - else: - machines_to_update.append(machine) - provision.create_if_vm( - ctx, - misc.canonicalize_hostname(machine), - ) - elif ctx.unlock: - if ctx.owner is None and user is None: - user = misc.get_user() - # If none of them are vpm, do them all in one shot - if not filter(lambda x: is_vm(status=x), machines): - res = unlock_many(machines, user) - return 0 if res else 1 - for machine in machines: - if not unlock_one(ctx, machine, user): - ret = 1 - if not ctx.f: - return ret - else: - machines_to_update.append(machine) - elif ctx.num_to_lock: - result = lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user, - ctx.desc, ctx.os_type, ctx.os_version, ctx.arch) - if not result: - ret = 1 - else: - machines_to_update = result.keys() - if ctx.machine_type == 'vps': - shortnames = ' '.join( - [misc.decanonicalize_hostname(name) for name in - result.keys()] - ) - if len(result) < ctx.num_to_lock: - log.error("Locking failed.") - for machine in result: - unlock_one(ctx, machine, user) - ret = 1 - else: - log.info("Successfully Locked:\n%s\n" % shortnames) - log.info( - "Unable to display keys at this time (virtual " + - "machines are booting).") - log.info( - "Please run teuthology-lock --list-targets %s once " + - "these machines come up.", - shortnames) - else: - print yaml.safe_dump( - dict(targets=result), - default_flow_style=False) - elif ctx.update: - assert ctx.desc is not None or ctx.status is not None, \ - 'you must specify description or status to update' - assert ctx.owner is None, 'only description and status may be updated' - machines_to_update = machines - - if ctx.desc is not None or ctx.status is not None: - for machine in machines_to_update: - update_lock(machine, ctx.desc, ctx.status) - - return ret - - -def lock_many_openstack(ctx, num, machine_type, user=None, description=None, - arch=None): - os_type = provision.get_distro(ctx) - os_version = provision.get_distro_version(ctx) - if hasattr(ctx, 'config'): - resources_hint = ctx.config.get('openstack') - else: - resources_hint = None - machines = provision.ProvisionOpenStack().create( - num, os_type, os_version, arch, resources_hint) - result = {} - for machine in machines: - lock_one(machine, user, description) - result[machine] = None # we do not collect ssh host keys yet - return result - -def lock_many(ctx, num, machine_type, user=None, description=None, - os_type=None, os_version=None, arch=None): - if user is None: - user = misc.get_user() - - if not vps_version_or_type_valid(ctx.machine_type, os_type, os_version): - log.error('Invalid os-type or version detected -- lock failed') - return - - # In the for loop below we can safely query for all bare-metal machine_type - # values at once. So, if we're being asked for 'plana,mira,burnupi', do it - # all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query - # for 'plana,mira,burnupi' and one for 'vps' - machine_types_list = misc.get_multi_machine_types(machine_type) - if machine_types_list == ['vps']: - machine_types = machine_types_list - elif machine_types_list == ['openstack']: - return lock_many_openstack(ctx, num, machine_type, - user=user, - description=description, - arch=arch) - elif 'vps' in machine_types_list: - machine_types_non_vps = list(machine_types_list) - machine_types_non_vps.remove('vps') - machine_types_non_vps = '|'.join(machine_types_non_vps) - machine_types = [machine_types_non_vps, 'vps'] - else: - machine_types_str = '|'.join(machine_types_list) - machine_types = [machine_types_str, ] - - for machine_type in machine_types: - uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '') - data = dict( - locked_by=user, - count=num, - machine_type=machine_type, - description=description, - ) - # Only query for os_type/os_version if non-vps and non-libcloud, since - # in that case we just create them. - if machine_type not in ['vps'] + provision.cloud.get_types(): - if os_type: - data['os_type'] = os_type - if os_version: - data['os_version'] = os_version - if arch: - data['arch'] = arch - log.debug("lock_many request: %s", repr(data)) - response = requests.post( - uri, - data=json.dumps(data), - headers={'content-type': 'application/json'}, - ) - if response.ok: - machines = {misc.canonicalize_hostname(machine['name']): - machine['ssh_pub_key'] for machine in response.json()} - log.debug('locked {machines}'.format( - machines=', '.join(machines.keys()))) - if machine_type in ['vps'] + provision.cloud.get_types(): - ok_machs = {} - for machine in machines: - if provision.create_if_vm(ctx, machine): - ok_machs[machine] = machines[machine] - else: - log.error('Unable to create virtual machine: %s', - machine) - unlock_one(ctx, machine, user) - return ok_machs - if machine_type in provision.cloud.get_types(): - machines = do_update_keys(machines.keys()) - return machines - elif response.status_code == 503: - log.error('Insufficient nodes available to lock %d %s nodes.', - num, machine_type) - log.error(response.text) - else: - log.error('Could not lock %d %s nodes, reason: unknown.', - num, machine_type) - return [] - - -def lock_one(name, user=None, description=None): - name = misc.canonicalize_hostname(name, user=None) - if user is None: - user = misc.get_user() - request = dict(name=name, locked=True, locked_by=user, - description=description) - uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '') - response = requests.put(uri, json.dumps(request)) - success = response.ok - if success: - log.debug('locked %s as %s', name, user) - else: - try: - reason = response.json().get('message') - except ValueError: - reason = str(response.status_code) - log.error('failed to lock {node}. reason: {reason}'.format( - node=name, reason=reason)) - return response - - -def unlock_many(names, user): - fixed_names = [misc.canonicalize_hostname(name, user=None) for name in - names] - names = fixed_names - uri = os.path.join(config.lock_server, 'nodes', 'unlock_many', '') - data = dict( - locked_by=user, - names=names, - ) - response = requests.post( - uri, - data=json.dumps(data), - headers={'content-type': 'application/json'}, - ) - if response.ok: - log.debug("Unlocked: %s", ', '.join(names)) - else: - log.error("Failed to unlock: %s", ', '.join(names)) - return response.ok - - -def unlock_one(ctx, name, user, description=None): - name = misc.canonicalize_hostname(name, user=None) - if not provision.destroy_if_vm(ctx, name, user, description): - log.error('destroy failed for %s', name) - request = dict(name=name, locked=False, locked_by=user, - description=description) - uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '') - with safe_while( - sleep=1, increment=0.5, action="unlock %s" % name) as proceed: - while proceed(): - try: - response = requests.put(uri, json.dumps(request)) - break - # Work around https://github.com/kennethreitz/requests/issues/2364 - except requests.ConnectionError as e: - log.warn("Saw %s while unlocking; retrying...", str(e)) - success = response.ok - if success: - log.info('unlocked %s', name) - else: - try: - reason = response.json().get('message') - except ValueError: - reason = str(response.status_code) - log.error('failed to unlock {node}. reason: {reason}'.format( - node=name, reason=reason)) - return success - -def locked_since_seconds(node): - now = datetime.datetime.now() - since = datetime.datetime.strptime( - node['locked_since'], '%Y-%m-%d %H:%M:%S.%f') - return (now - since).total_seconds() - -def list_locks(keyed_by_name=False, **kwargs): - uri = os.path.join(config.lock_server, 'nodes', '') - for key, value in kwargs.iteritems(): - if kwargs[key] is False: - kwargs[key] = '0' - if kwargs[key] is True: - kwargs[key] = '1' - if kwargs: - if 'machine_type' in kwargs: - kwargs['machine_type'] = kwargs['machine_type'].replace(',','|') - uri += '?' + urllib.urlencode(kwargs) - try: - response = requests.get(uri) - except requests.ConnectionError: - success = False - log.exception("Could not contact lock server: %s", config.lock_server) - else: - success = response.ok - if success: - if not keyed_by_name: - return response.json() - else: - return {node['name']: node - for node in response.json()} - return dict() - - -def find_stale_locks(owner=None): - """ - Return a list of node dicts corresponding to nodes that were locked to run - a job, but the job is no longer running. The purpose of this is to enable - us to nuke nodes that were left locked due to e.g. infrastructure failures - and return them to the pool. - - :param owner: If non-None, return nodes locked by owner. Default is None. - """ - def might_be_stale(node_dict): - """ - Answer the question: "might this be a stale lock?" - - The answer is yes if: - It is locked - It has a non-null description containing multiple '/' characters - - ... because we really want "nodes that were locked for a particular job - and are still locked" and the above is currently the best way to guess. - """ - desc = node_dict['description'] - if (node_dict['locked'] is True and - desc is not None and desc.startswith('/') and - desc.count('/') > 1): - return True - return False - - # Which nodes are locked for jobs? - nodes = list_locks(locked=True) - if owner is not None: - nodes = [node for node in nodes if node['locked_by'] == owner] - nodes = filter(might_be_stale, nodes) - - def node_job_is_active(node, cache): - """ - Is this node's job active (e.g. running or waiting)? - - :param node: The node dict as returned from the lock server - :param cache: A set() used for caching results - :returns: True or False - """ - description = node['description'] - if description in cache: - return True - (name, job_id) = description.split('/')[-2:] - url = os.path.join(config.results_server, 'runs', name, 'jobs', job_id, - '') - resp = requests.get(url) - job_info = resp.json() - if job_info['status'] in ('running', 'waiting'): - cache.add(description) - return True - return False - - result = list() - # Here we build the list of of nodes that are locked, for a job (as opposed - # to being locked manually for random monkeying), where the job is not - # running - active_jobs = set() - for node in nodes: - if node_job_is_active(node, active_jobs): - continue - result.append(node) - return result - - -def update_lock(name, description=None, status=None, ssh_pub_key=None): - name = misc.canonicalize_hostname(name, user=None) - updated = {} - if description is not None: - updated['description'] = description - if status is not None: - updated['up'] = (status == 'up') - if ssh_pub_key is not None: - updated['ssh_pub_key'] = ssh_pub_key - - if updated: - uri = os.path.join(config.lock_server, 'nodes', name, '') - response = requests.put( - uri, - json.dumps(updated)) - return response.ok - return True - - -def update_inventory(node_dict): - """ - Like update_lock(), but takes a dict and doesn't try to do anything smart - by itself - """ - name = node_dict.get('name') - if not name: - raise ValueError("must specify name") - if not config.lock_server: - return - uri = os.path.join(config.lock_server, 'nodes', name, '') - log.info("Updating %s on lock server", name) - response = requests.put( - uri, - json.dumps(node_dict), - headers={'content-type': 'application/json'}, - ) - if response.status_code == 404: - log.info("Creating new node %s on lock server", name) - uri = os.path.join(config.lock_server, 'nodes', '') - response = requests.post( - uri, - json.dumps(node_dict), - headers={'content-type': 'application/json'}, - ) - if not response.ok: - log.error("Node update/creation failed for %s: %s", - name, response.text) - return response.ok - - -def updatekeys(args): - loglevel = logging.DEBUG if args['--verbose'] else logging.INFO - logging.basicConfig( - level=loglevel, - ) - all_ = args['--all'] - if all_: - machines = [] - elif args['']: - machines = [misc.canonicalize_hostname(m, user=None) - for m in args['']] - elif args['--targets']: - targets = args['--targets'] - with file(targets) as f: - docs = yaml.safe_load_all(f) - for doc in docs: - machines = [n for n in doc.get('targets', dict()).iterkeys()] - - return do_update_keys(machines, all_) - - -def do_update_keys(machines, all_=False): - reference = list_locks(keyed_by_name=True) - if all_: - machines = reference.keys() - keys_dict = misc.ssh_keyscan(machines) - return push_new_keys(keys_dict, reference) - - -def push_new_keys(keys_dict, reference): - ret = 0 - for hostname, pubkey in keys_dict.iteritems(): - log.info('Checking %s', hostname) - if reference[hostname]['ssh_pub_key'] != pubkey: - log.info('New key found. Updating...') - if not update_lock(hostname, ssh_pub_key=pubkey): - log.error('failed to update %s!', hostname) - ret = 1 - return ret - - -def do_summary(ctx): - lockd = collections.defaultdict(lambda: [0, 0, 'unknown']) - if ctx.machine_type: - locks = list_locks(machine_type=ctx.machine_type) - else: - locks = list_locks() - for l in locks: - who = l['locked_by'] if l['locked'] == 1 \ - else '(free)', l['machine_type'] - lockd[who][0] += 1 - lockd[who][1] += 1 if l['up'] else 0 - lockd[who][2] = l['machine_type'] - - locks = sorted([p for p in lockd.iteritems() - ], key=lambda sort: (sort[1][2], sort[1][0])) - total_count, total_up = 0, 0 - print "TYPE COUNT UP OWNER" - - for (owner, (count, upcount, machinetype)) in locks: - # if machinetype == spectype: - print "{machinetype:8s} {count:3d} {up:3d} {owner}".format( - count=count, up=upcount, owner=owner[0], - machinetype=machinetype) - total_count += count - total_up += upcount - - print " --- ---" - print "{cnt:12d} {up:3d}".format(cnt=total_count, up=total_up) diff --git a/teuthology/lock/__init__.py b/teuthology/lock/__init__.py new file mode 100644 index 0000000000..5a3fae1c9c --- /dev/null +++ b/teuthology/lock/__init__.py @@ -0,0 +1 @@ +from teuthology.lock import cli, keys, ops, query, util # noqa \ No newline at end of file diff --git a/teuthology/lock/cli.py b/teuthology/lock/cli.py new file mode 100644 index 0000000000..75869871b8 --- /dev/null +++ b/teuthology/lock/cli.py @@ -0,0 +1,266 @@ +import argparse +import collections +import json +import logging +import re + +import yaml + +import teuthology +import teuthology.provision +from teuthology import misc +from teuthology.config import set_config_attr + +from teuthology.lock import ( + keys, + ops, + util, + query, +) + + +log = logging.getLogger(__name__) + + +def main(ctx): + if ctx.verbose: + teuthology.log.setLevel(logging.DEBUG) + + set_config_attr(ctx) + + ret = 0 + user = ctx.owner + machines = [misc.canonicalize_hostname(m, user=False) + for m in ctx.machines] + machines_to_update = [] + + if ctx.targets: + try: + with file(ctx.targets) as f: + g = yaml.safe_load_all(f) + for new in g: + if 'targets' in new: + for t in new['targets'].iterkeys(): + machines.append(t) + except IOError as e: + raise argparse.ArgumentTypeError(str(e)) + + if ctx.f: + assert ctx.lock or ctx.unlock, \ + '-f is only supported by --lock and --unlock' + if machines: + assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \ + or ctx.update or ctx.brief, \ + 'machines cannot be specified with that operation' + else: + if ctx.lock: + log.error("--lock requires specific machines passed as arguments") + else: + # This condition might never be hit, but it's not clear. + assert ctx.num_to_lock or ctx.list or ctx.list_targets or \ + ctx.summary or ctx.brief, \ + 'machines must be specified for that operation' + if ctx.all: + assert ctx.list or ctx.list_targets or ctx.brief, \ + '--all can only be used with --list, --list-targets, and --brief' + assert ctx.owner is None, \ + '--all and --owner are mutually exclusive' + assert not machines, \ + '--all and listing specific machines are incompatible' + if ctx.num_to_lock: + assert ctx.machine_type, \ + 'must specify machine type to lock' + + if ctx.brief or ctx.list or ctx.list_targets: + assert ctx.desc is None, '--desc does nothing with --list/--brief' + + # we may need to update host keys for vms. Don't do it for + # every vm; however, update any vms included in the list given + # to the CLI (machines), or any owned by the specified owner or + # invoking user if no machines are specified. + vmachines = [] + statuses = query.get_statuses(machines) + owner = ctx.owner or misc.get_user() + for machine in statuses: + if query.is_vm(status=machine) and machine['locked'] and \ + (machines or machine['locked_by'] == owner): + vmachines.append(machine['name']) + if vmachines: + log.info("updating host keys for %s", ' '.join(sorted(vmachines))) + keys.do_update_keys(vmachines) + # get statuses again to refresh any updated keys + statuses = query.get_statuses(machines) + if statuses: + statuses = util.winnow(statuses, ctx.machine_type, 'machine_type') + if not machines and ctx.owner is None and not ctx.all: + ctx.owner = misc.get_user() + statuses = util.winnow(statuses, ctx.owner, 'locked_by') + statuses = util.winnow(statuses, ctx.status, 'up', + lambda s: s['up'] == (ctx.status == 'up')) + statuses = util.winnow(statuses, ctx.locked, 'locked', + lambda s: s['locked'] == (ctx.locked == 'true')) + statuses = util.winnow(statuses, ctx.desc, 'description') + statuses = util.winnow(statuses, ctx.desc_pattern, 'description', + lambda s: s['description'] and \ + ctx.desc_pattern in s['description']) + if ctx.json_query: + statuses = util.json_matching_statuses(ctx.json_query, statuses) + statuses = util.winnow(statuses, ctx.os_type, 'os_type') + statuses = util.winnow(statuses, ctx.os_version, 'os_version') + + # When listing, only show the vm_host's name, not every detail + for s in statuses: + if not query.is_vm(status=s): + continue + # with an OpenStack API, there is no host for a VM + if s['vm_host'] is None: + continue + vm_host_name = s.get('vm_host', dict())['name'] + if vm_host_name: + s['vm_host'] = vm_host_name + if ctx.list: + print json.dumps(statuses, indent=4) + + elif ctx.brief: + for s in sorted(statuses, key=lambda s: s.get('name')): + locked = "un" if s['locked'] == 0 else " " + mo = re.match('\w+@(\w+?)\..*', s['name']) + host = mo.group(1) if mo else s['name'] + print '{host} {locked}locked {owner} "{desc}"'.format( + locked=locked, host=host, + owner=s['locked_by'], desc=s['description']) + + else: + frag = {'targets': {}} + for f in statuses: + frag['targets'][f['name']] = f['ssh_pub_key'] + print yaml.safe_dump(frag, default_flow_style=False) + else: + log.error('error retrieving lock statuses') + ret = 1 + + elif ctx.summary: + do_summary(ctx) + return 0 + + elif ctx.lock: + if not util.vps_version_or_type_valid(ctx.machine_type, ctx.os_type, + ctx.os_version): + log.error('Invalid os-type or version detected -- lock failed') + return 1 + for machine in machines: + if not ops.lock_one(machine, user, ctx.desc): + ret = 1 + if not ctx.f: + return ret + else: + machines_to_update.append(machine) + teuthology.provision.create_if_vm( + ctx, + misc.canonicalize_hostname(machine), + ) + elif ctx.unlock: + if ctx.owner is None and user is None: + user = misc.get_user() + # If none of them are vpm, do them all in one shot + if not filter(query.is_vm, machines): + res = ops.unlock_many(machines, user) + return 0 if res else 1 + for machine in machines: + if not ops.unlock_one(ctx, machine, user): + ret = 1 + if not ctx.f: + return ret + else: + machines_to_update.append(machine) + elif ctx.num_to_lock: + result = ops.lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user, + ctx.desc, ctx.os_type, ctx.os_version, ctx.arch) + if not result: + ret = 1 + else: + machines_to_update = result.keys() + if ctx.machine_type == 'vps': + shortnames = ' '.join( + [misc.decanonicalize_hostname(name) for name in + result.keys()] + ) + if len(result) < ctx.num_to_lock: + log.error("Locking failed.") + for machine in result: + ops.unlock_one(ctx, machine, user) + ret = 1 + else: + log.info("Successfully Locked:\n%s\n" % shortnames) + log.info( + "Unable to display keys at this time (virtual " + + "machines are booting).") + log.info( + "Please run teuthology-lock --list-targets %s once " + + "these machines come up.", + shortnames) + else: + print yaml.safe_dump( + dict(targets=result), + default_flow_style=False) + elif ctx.update: + assert ctx.desc is not None or ctx.status is not None, \ + 'you must specify description or status to update' + assert ctx.owner is None, 'only description and status may be updated' + machines_to_update = machines + + if ctx.desc is not None or ctx.status is not None: + for machine in machines_to_update: + ops.update_lock(machine, ctx.desc, ctx.status) + + return ret + +def do_summary(ctx): + lockd = collections.defaultdict(lambda: [0, 0, 'unknown']) + if ctx.machine_type: + locks = query.list_locks(machine_type=ctx.machine_type) + else: + locks = query.list_locks() + for l in locks: + who = l['locked_by'] if l['locked'] == 1 \ + else '(free)', l['machine_type'] + lockd[who][0] += 1 + lockd[who][1] += 1 if l['up'] else 0 + lockd[who][2] = l['machine_type'] + + locks = sorted([p for p in lockd.iteritems() + ], key=lambda sort: (sort[1][2], sort[1][0])) + total_count, total_up = 0, 0 + print "TYPE COUNT UP OWNER" + + for (owner, (count, upcount, machinetype)) in locks: + # if machinetype == spectype: + print "{machinetype:8s} {count:3d} {up:3d} {owner}".format( + count=count, up=upcount, owner=owner[0], + machinetype=machinetype) + total_count += count + total_up += upcount + + print " --- ---" + print "{cnt:12d} {up:3d}".format(cnt=total_count, up=total_up) + + +def updatekeys(args): + loglevel = logging.DEBUG if args['--verbose'] else logging.INFO + logging.basicConfig( + level=loglevel, + ) + all_ = args['--all'] + if all_: + machines = [] + elif args['']: + machines = [misc.canonicalize_hostname(m, user=None) + for m in args['']] + elif args['--targets']: + targets = args['--targets'] + with file(targets) as f: + docs = yaml.safe_load_all(f) + for doc in docs: + machines = [n for n in doc.get('targets', dict()).iterkeys()] + + return keys.do_update_keys(machines, all_) \ No newline at end of file diff --git a/teuthology/lock/keys.py b/teuthology/lock/keys.py new file mode 100644 index 0000000000..b6fe58ec2d --- /dev/null +++ b/teuthology/lock/keys.py @@ -0,0 +1,27 @@ +import logging + +from teuthology import misc + +from . import ops, query + +log = logging.getLogger(__name__) + + +def do_update_keys(machines, all_=False): + reference = query.list_locks(keyed_by_name=True) + if all_: + machines = reference.keys() + keys_dict = misc.ssh_keyscan(machines) + return push_new_keys(keys_dict, reference) + + +def push_new_keys(keys_dict, reference): + ret = 0 + for hostname, pubkey in keys_dict.iteritems(): + log.info('Checking %s', hostname) + if reference[hostname]['ssh_pub_key'] != pubkey: + log.info('New key found. Updating...') + if not ops.update_lock(hostname, ssh_pub_key=pubkey): + log.error('failed to update %s!', hostname) + ret = 1 + return ret \ No newline at end of file diff --git a/teuthology/lock/ops.py b/teuthology/lock/ops.py new file mode 100644 index 0000000000..ccd7754035 --- /dev/null +++ b/teuthology/lock/ops.py @@ -0,0 +1,237 @@ +import logging +import json +import os + +import requests + +import teuthology.provision +from teuthology import misc +from teuthology.config import config +from teuthology.contextutil import safe_while + +import util +import keys + +log = logging.getLogger(__name__) + + +def lock_many_openstack(ctx, num, machine_type, user=None, description=None, + arch=None): + os_type = teuthology.provision.get_distro(ctx) + os_version = teuthology.provision.get_distro_version(ctx) + if hasattr(ctx, 'config'): + resources_hint = ctx.config.get('openstack') + else: + resources_hint = None + machines = teuthology.provision.openstack.ProvisionOpenStack().create( + num, os_type, os_version, arch, resources_hint) + result = {} + for machine in machines: + lock_one(machine, user, description) + result[machine] = None # we do not collect ssh host keys yet + return result + + +def lock_many(ctx, num, machine_type, user=None, description=None, + os_type=None, os_version=None, arch=None): + if user is None: + user = misc.get_user() + + if not util.vps_version_or_type_valid( + ctx.machine_type, + os_type, + os_version + ): + log.error('Invalid os-type or version detected -- lock failed') + return + + # In the for loop below we can safely query for all bare-metal machine_type + # values at once. So, if we're being asked for 'plana,mira,burnupi', do it + # all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query + # for 'plana,mira,burnupi' and one for 'vps' + machine_types_list = misc.get_multi_machine_types(machine_type) + if machine_types_list == ['vps']: + machine_types = machine_types_list + elif machine_types_list == ['openstack']: + return lock_many_openstack(ctx, num, machine_type, + user=user, + description=description, + arch=arch) + elif 'vps' in machine_types_list: + machine_types_non_vps = list(machine_types_list) + machine_types_non_vps.remove('vps') + machine_types_non_vps = '|'.join(machine_types_non_vps) + machine_types = [machine_types_non_vps, 'vps'] + else: + machine_types_str = '|'.join(machine_types_list) + machine_types = [machine_types_str, ] + + for machine_type in machine_types: + uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '') + data = dict( + locked_by=user, + count=num, + machine_type=machine_type, + description=description, + ) + # Only query for os_type/os_version if non-vps and non-libcloud, since + # in that case we just create them. + vm_types = ['vps'] + teuthology.provision.cloud.get_types() + if machine_type not in vm_types: + if os_type: + data['os_type'] = os_type + if os_version: + data['os_version'] = os_version + if arch: + data['arch'] = arch + log.debug("lock_many request: %s", repr(data)) + response = requests.post( + uri, + data=json.dumps(data), + headers={'content-type': 'application/json'}, + ) + if response.ok: + machines = {misc.canonicalize_hostname(machine['name']): + machine['ssh_pub_key'] for machine in response.json()} + log.debug('locked {machines}'.format( + machines=', '.join(machines.keys()))) + if machine_type in vm_types: + ok_machs = {} + for machine in machines: + if teuthology.provision.create_if_vm(ctx, machine): + ok_machs[machine] = machines[machine] + else: + log.error('Unable to create virtual machine: %s', + machine) + unlock_one(ctx, machine, user) + return ok_machs + return machines + elif response.status_code == 503: + log.error('Insufficient nodes available to lock %d %s nodes.', + num, machine_type) + log.error(response.text) + else: + log.error('Could not lock %d %s nodes, reason: unknown.', + num, machine_type) + return [] + + +def lock_one(name, user=None, description=None): + name = misc.canonicalize_hostname(name, user=None) + if user is None: + user = misc.get_user() + request = dict(name=name, locked=True, locked_by=user, + description=description) + uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '') + response = requests.put(uri, json.dumps(request)) + success = response.ok + if success: + log.debug('locked %s as %s', name, user) + else: + try: + reason = response.json().get('message') + except ValueError: + reason = str(response.status_code) + log.error('failed to lock {node}. reason: {reason}'.format( + node=name, reason=reason)) + return response + + +def unlock_many(names, user): + fixed_names = [misc.canonicalize_hostname(name, user=None) for name in + names] + names = fixed_names + uri = os.path.join(config.lock_server, 'nodes', 'unlock_many', '') + data = dict( + locked_by=user, + names=names, + ) + response = requests.post( + uri, + data=json.dumps(data), + headers={'content-type': 'application/json'}, + ) + if response.ok: + log.debug("Unlocked: %s", ', '.join(names)) + else: + log.error("Failed to unlock: %s", ', '.join(names)) + return response.ok + + +def unlock_one(ctx, name, user, description=None): + name = misc.canonicalize_hostname(name, user=None) + if not teuthology.provision.destroy_if_vm(ctx, name, user, description): + log.error('destroy failed for %s', name) + request = dict(name=name, locked=False, locked_by=user, + description=description) + uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '') + with safe_while( + sleep=1, increment=0.5, action="unlock %s" % name) as proceed: + while proceed(): + try: + response = requests.put(uri, json.dumps(request)) + break + # Work around https://github.com/kennethreitz/requests/issues/2364 + except requests.ConnectionError as e: + log.warn("Saw %s while unlocking; retrying...", str(e)) + success = response.ok + if success: + log.info('unlocked %s', name) + else: + try: + reason = response.json().get('message') + except ValueError: + reason = str(response.status_code) + log.error('failed to unlock {node}. reason: {reason}'.format( + node=name, reason=reason)) + return success + + +def update_lock(name, description=None, status=None, ssh_pub_key=None): + name = misc.canonicalize_hostname(name, user=None) + updated = {} + if description is not None: + updated['description'] = description + if status is not None: + updated['up'] = (status == 'up') + if ssh_pub_key is not None: + updated['ssh_pub_key'] = ssh_pub_key + + if updated: + uri = os.path.join(config.lock_server, 'nodes', name, '') + response = requests.put( + uri, + json.dumps(updated)) + return response.ok + return True + + +def update_inventory(node_dict): + """ + Like update_lock(), but takes a dict and doesn't try to do anything smart + by itself + """ + name = node_dict.get('name') + if not name: + raise ValueError("must specify name") + if not config.lock_server: + return + uri = os.path.join(config.lock_server, 'nodes', name, '') + log.info("Updating %s on lock server", name) + response = requests.put( + uri, + json.dumps(node_dict), + headers={'content-type': 'application/json'}, + ) + if response.status_code == 404: + log.info("Creating new node %s on lock server", name) + uri = os.path.join(config.lock_server, 'nodes', '') + response = requests.post( + uri, + json.dumps(node_dict), + headers={'content-type': 'application/json'}, + ) + if not response.ok: + log.error("Node update/creation failed for %s: %s", + name, response.text) + return response.ok diff --git a/teuthology/lock/query.py b/teuthology/lock/query.py new file mode 100644 index 0000000000..b6ef1cedbf --- /dev/null +++ b/teuthology/lock/query.py @@ -0,0 +1,141 @@ +import logging +import os +import urllib + +import requests + +from teuthology import misc +from teuthology.config import config + + +log = logging.getLogger(__name__) + + +def get_status(name): + name = misc.canonicalize_hostname(name, user=None) + uri = os.path.join(config.lock_server, 'nodes', name, '') + response = requests.get(uri) + success = response.ok + if success: + return response.json() + log.warning( + "Failed to query lock server for status of {name}".format(name=name)) + return None + + +def get_statuses(machines): + if machines: + statuses = [] + for machine in machines: + machine = misc.canonicalize_hostname(machine) + status = get_status(machine) + if status: + statuses.append(status) + else: + log.error("Lockserver doesn't know about machine: %s" % + machine) + else: + statuses = list_locks() + return statuses + + +def is_vm(name=None, status=None): + if status is None: + if name is None: + raise ValueError("Must provide either name or status, or both") + name = misc.canonicalize_hostname(name) + status = get_status(name) + return status.get('is_vm', False) + + +def list_locks(keyed_by_name=False, **kwargs): + uri = os.path.join(config.lock_server, 'nodes', '') + for key, value in kwargs.iteritems(): + if kwargs[key] is False: + kwargs[key] = '0' + if kwargs[key] is True: + kwargs[key] = '1' + if kwargs: + if 'machine_type' in kwargs: + kwargs['machine_type'] = kwargs['machine_type'].replace(',','|') + uri += '?' + urllib.urlencode(kwargs) + try: + response = requests.get(uri) + except requests.ConnectionError: + success = False + log.exception("Could not contact lock server: %s", config.lock_server) + else: + success = response.ok + if success: + if not keyed_by_name: + return response.json() + else: + return {node['name']: node + for node in response.json()} + return dict() + + +def find_stale_locks(owner=None): + """ + Return a list of node dicts corresponding to nodes that were locked to run + a job, but the job is no longer running. The purpose of this is to enable + us to nuke nodes that were left locked due to e.g. infrastructure failures + and return them to the pool. + + :param owner: If non-None, return nodes locked by owner. Default is None. + """ + def might_be_stale(node_dict): + """ + Answer the question: "might this be a stale lock?" + + The answer is yes if: + It is locked + It has a non-null description containing multiple '/' characters + + ... because we really want "nodes that were locked for a particular job + and are still locked" and the above is currently the best way to guess. + """ + desc = node_dict['description'] + if (node_dict['locked'] is True and + desc is not None and desc.startswith('/') and + desc.count('/') > 1): + return True + return False + + # Which nodes are locked for jobs? + nodes = list_locks(locked=True) + if owner is not None: + nodes = [node for node in nodes if node['locked_by'] == owner] + nodes = filter(might_be_stale, nodes) + + def node_job_is_active(node, cache): + """ + Is this node's job active (e.g. running or waiting)? + + :param node: The node dict as returned from the lock server + :param cache: A set() used for caching results + :returns: True or False + """ + description = node['description'] + if description in cache: + return True + (name, job_id) = description.split('/')[-2:] + url = os.path.join(config.results_server, 'runs', name, 'jobs', job_id, + '') + resp = requests.get(url) + job_info = resp.json() + if job_info['status'] in ('running', 'waiting'): + cache.add(description) + return True + return False + + result = list() + # Here we build the list of of nodes that are locked, for a job (as opposed + # to being locked manually for random monkeying), where the job is not + # running + active_jobs = set() + for node in nodes: + if node_job_is_active(node, active_jobs): + continue + result.append(node) + return result \ No newline at end of file diff --git a/teuthology/lock/test/__init__.py b/teuthology/lock/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/teuthology/test/test_lock.py b/teuthology/lock/test/test_lock.py similarity index 57% rename from teuthology/test/test_lock.py rename to teuthology/lock/test/test_lock.py index 9dd9e442f3..5f1679afc3 100644 --- a/teuthology/test/test_lock.py +++ b/teuthology/lock/test/test_lock.py @@ -1,8 +1,7 @@ -from teuthology import lock - +import teuthology.lock.util class TestLock(object): def test_locked_since_seconds(self): node = { "locked_since": "2013-02-07 19:33:55.000000" } - assert lock.locked_since_seconds(node) > 3600 + assert teuthology.lock.util.locked_since_seconds(node) > 3600 diff --git a/teuthology/lock/util.py b/teuthology/lock/util.py new file mode 100644 index 0000000000..5dc0814d49 --- /dev/null +++ b/teuthology/lock/util.py @@ -0,0 +1,121 @@ +import datetime +import json +import logging + +from teuthology import misc +import teuthology.provision.downburst + +log = logging.getLogger(__name__) + + +def vps_version_or_type_valid(machine_type, os_type, os_version): + """ + Check os-type and os-version parameters when locking a vps. + Os-type will always be set (defaults to ubuntu). + + In the case where downburst does not handle list-json (an older version + of downburst, for instance), a message is printed and this checking + is skipped (so that this code should behave as it did before this + check was added). + """ + if not machine_type == 'vps': + return True + if os_type is None or os_version is None: + # we'll use the defaults provided by provision.create_if_vm + # later on during provisioning + return True + valid_os_and_version = \ + teuthology.provision.downburst.get_distro_from_downburst() + if os_type not in valid_os_and_version: + log.error("os-type '%s' is invalid. Try one of: %s", + os_type, + ', '.join(valid_os_and_version.keys())) + return False + if not validate_distro_version(os_version, + valid_os_and_version[os_type]): + log.error( + "os-version '%s' is invalid for os-type '%s'. Try one of: %s", + os_version, + os_type, + ', '.join(valid_os_and_version[os_type])) + return False + return True + + +def validate_distro_version(version, supported_versions): + """ + Return True if the version is valid. For Ubuntu, possible + supported version values are of the form '12.04 (precise)' where + either the number of the version name is acceptable. + """ + if version in supported_versions: + return True + for parts in supported_versions: + part = parts.split('(') + if len(part) == 2: + if version == part[0]: + return True + if version == part[1][0:len(part[1])-1]: + return True + + +def json_matching_statuses(json_file_or_str, statuses): + """ + Filter statuses by json dict in file or fragment; return list of + matching statuses. json_file_or_str must be a file containing + json or json in a string. + """ + try: + open(json_file_or_str, 'r') + except IOError: + query = json.loads(json_file_or_str) + else: + query = json.load(json_file_or_str) + + if not isinstance(query, dict): + raise RuntimeError('--json-query must be a dict') + + return_statuses = list() + for status in statuses: + for k, v in query.iteritems(): + if not misc.is_in_dict(k, v, status): + break + else: + return_statuses.append(status) + + return return_statuses + + +def winnow(statuses, arg, status_key, func=None): + """ + Call with a list of statuses, and the ctx. + 'arg' that you may want to filter by. + If arg is not None, filter statuses by either: + + 1) func=None: filter by status[status_key] == arg + remove any status that fails + + 2) func=: remove any + status for which func returns False + + Return the possibly-smaller set of statuses. + """ + + if arg is not None: + if func: + statuses = [_status for _status in statuses + if func(_status)] + else: + statuses = [_status for _status in statuses + if _status[status_key] == arg] + + return statuses + + +def locked_since_seconds(node): + now = datetime.datetime.now() + since = datetime.datetime.strptime( + node['locked_since'], '%Y-%m-%d %H:%M:%S.%f') + return (now - since).total_seconds() + + diff --git a/teuthology/misc.py b/teuthology/misc.py index 9929ca26dd..6600ba88d9 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -347,7 +347,7 @@ def roles_of_type(roles_for_host, type_): Generator of ids. Each call returns the next possible role of the type specified. - :param roles_for host: list of roles possible + :param roles_for_host: list of roles possible :param type_: type of role """ for role in cluster_roles_of_type(roles_for_host, type_, None): @@ -360,7 +360,7 @@ def cluster_roles_of_type(roles_for_host, type_, cluster): Generator of roles. Each call returns the next possible role of the type specified. - :param roles_for host: list of roles possible + :param roles_for_host: list of roles possible :param type_: type of role :param cluster: cluster name """ @@ -388,7 +388,7 @@ def all_roles_of_type(cluster, type_): type specified. :param cluster: Cluster extracted from the ctx. - :type_: role type + :param type_: role type """ for _, roles_for_host in cluster.remotes.iteritems(): for id_ in roles_of_type(roles_for_host, type_): diff --git a/teuthology/nuke/__init__.py b/teuthology/nuke/__init__.py index 071acc238d..605072d7ae 100644 --- a/teuthology/nuke/__init__.py +++ b/teuthology/nuke/__init__.py @@ -4,15 +4,23 @@ import json import logging import os import subprocess + import yaml import teuthology - -from ..config import config, FakeNamespace -from ..lock import ( - list_locks, locked_since_seconds, unlock_one, find_stale_locks, get_status, - is_vm +from teuthology.lock.ops import unlock_one +from teuthology.lock.query import is_vm, list_locks, \ + find_stale_locks +from teuthology.lock.util import locked_since_seconds +from .actions import ( + check_console, clear_firewall, shutdown_daemons, remove_installed_packages, + reboot, remove_osd_mounts, remove_osd_tmpfs, kill_hadoop, + remove_ceph_packages, synch_clocks, unlock_firmware_repo, + remove_configuration_files, undo_multipath, reset_syslog_dir, + remove_ceph_data, remove_testing_tree, remove_yum_timedhosts, + kill_valgrind, ) +from ..config import config, FakeNamespace from ..misc import ( canonicalize_hostname, config_file, decanonicalize_hostname, merge_configs, get_user, sh @@ -22,15 +30,6 @@ from ..orchestra.remote import Remote from ..parallel import parallel from ..task.internal import check_lock, add_remotes, connect -from .actions import ( - check_console, clear_firewall, shutdown_daemons, remove_installed_packages, - reboot, remove_osd_mounts, remove_osd_tmpfs, kill_hadoop, - remove_ceph_packages, synch_clocks, - unlock_firmware_repo, remove_configuration_files, undo_multipath, - reset_syslog_dir, remove_ceph_data, remove_testing_tree, - remove_yum_timedhosts, kill_valgrind, -) - log = logging.getLogger(__name__) diff --git a/teuthology/openstack/test/openstack-integration.py b/teuthology/openstack/test/openstack-integration.py index 25695a1f7e..95f46e80b6 100644 --- a/teuthology/openstack/test/openstack-integration.py +++ b/teuthology/openstack/test/openstack-integration.py @@ -30,6 +30,9 @@ import tempfile import shutil import teuthology.lock +import teuthology.lock.cli +import teuthology.lock.query +import teuthology.lock.util import teuthology.nuke import teuthology.misc import teuthology.schedule @@ -144,7 +147,7 @@ class TestSuite(Integration): self.wait_worker() log = self.get_teuthology_log() assert "teuthology.run:FAIL" in log - locks = teuthology.lock.list_locks(locked=True) + locks = teuthology.lock.query.list_locks(locked=True) assert len(locks) == 0 class TestSchedule(Integration): @@ -221,7 +224,7 @@ class TestLock(Integration): def test_main(self): args = scripts.lock.parse_args(self.options + ['--lock']) - assert teuthology.lock.main(args) == 0 + assert teuthology.lock.cli.main(args) == 0 def test_lock_unlock(self): for image in teuthology.openstack.OpenStack.image2url.keys(): @@ -230,16 +233,16 @@ class TestLock(Integration): ['--lock-many', '1', '--os-type', os_type, '--os-version', os_version]) - assert teuthology.lock.main(args) == 0 - locks = teuthology.lock.list_locks(locked=True) + assert teuthology.lock.cli.main(args) == 0 + locks = teuthology.lock.query.list_locks(locked=True) assert len(locks) == 1 args = scripts.lock.parse_args(self.options + ['--unlock', locks[0]['name']]) - assert teuthology.lock.main(args) == 0 + assert teuthology.lock.cli.main(args) == 0 def test_list(self, capsys): args = scripts.lock.parse_args(self.options + ['--list', '--all']) - teuthology.lock.main(args) + teuthology.lock.cli.main(args) out, err = capsys.readouterr() assert 'machine_type' in out assert 'openstack' in out @@ -258,8 +261,8 @@ class TestNuke(Integration): ['--lock-many', '1', '--os-type', os_type, '--os-version', os_version]) - assert teuthology.lock.main(args) == 0 - locks = teuthology.lock.list_locks(locked=True) + assert teuthology.lock.cli.main(args) == 0 + locks = teuthology.lock.query.list_locks(locked=True) logging.info('list_locks = ' + str(locks)) assert len(locks) == 1 ctx = argparse.Namespace(name=None, @@ -269,5 +272,5 @@ class TestNuke(Integration): owner=locks[0]['locked_by'], teuthology_config={}) teuthology.nuke.nuke(ctx, should_unlock=True) - locks = teuthology.lock.list_locks(locked=True) + locks = teuthology.lock.query.list_locks(locked=True) assert len(locks) == 0 diff --git a/teuthology/orchestra/console.py b/teuthology/orchestra/console.py index 4799715b31..f12b4cacb1 100644 --- a/teuthology/orchestra/console.py +++ b/teuthology/orchestra/console.py @@ -6,7 +6,8 @@ import subprocess import sys import time -from teuthology import lock +import teuthology.lock.query +import teuthology.lock.util from teuthology.config import config from ..exceptions import ConsoleError @@ -316,9 +317,9 @@ class VirtualConsole(): raise RuntimeError("libvirt not found") self.shortname = remote.getShortName(name) - status_info = lock.get_status(self.shortname) + status_info = teuthology.lock.query.get_status(self.shortname) try: - if lock.is_vm(status=status_info): + if teuthology.lock.query.is_vm(status=status_info): phys_host = status_info['vm_host']['name'].split('.')[0] except TypeError: return diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 473d31121d..572f4ad315 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -1,6 +1,8 @@ """ Support for paramiko remote objects. """ +import teuthology.lock.query +import teuthology.lock.util from . import run from .opsys import OS import connection @@ -16,8 +18,6 @@ import netaddr import console -from teuthology import lock - log = logging.getLogger(__name__) @@ -140,7 +140,7 @@ class Remote(object): @property def machine_type(self): if not getattr(self, '_machine_type', None): - remote_info = lock.get_status(self.hostname) + remote_info = teuthology.lock.query.get_status(self.hostname) if not remote_info: return None self._machine_type = remote_info.get("machine_type", None) @@ -244,7 +244,7 @@ class Remote(object): """ if self.os.package_type != 'rpm': return - if lock.is_vm(self.shortname): + if teuthology.lock.query.is_vm(self.shortname): return self.run(args="sudo chcon {con} {path}".format( con=context, path=file_path)) @@ -448,7 +448,7 @@ class Remote(object): @property def is_vm(self): if not hasattr(self, '_is_vm'): - self._is_vm = lock.is_vm(self.name) + self._is_vm = teuthology.lock.query.is_vm(self.name) return self._is_vm def __del__(self): @@ -470,7 +470,7 @@ def getRemoteConsole(name, ipmiuser=None, ipmipass=None, ipmidomain=None, """ Return either VirtualConsole or PhysicalConsole depending on name. """ - if lock.is_vm(name): + if teuthology.lock.query.is_vm(name): return console.VirtualConsole(name) return console.PhysicalConsole( name, ipmiuser, ipmipass, ipmidomain, logfile, timeout) diff --git a/teuthology/provision/__init__.py b/teuthology/provision/__init__.py index 02253a501f..8c81e5bd27 100644 --- a/teuthology/provision/__init__.py +++ b/teuthology/provision/__init__.py @@ -1,11 +1,11 @@ import logging -from ..misc import decanonicalize_hostname, get_distro, get_distro_version -from ..lock import get_status, is_vm -from .downburst import Downburst -from .openstack import ProvisionOpenStack +import teuthology.lock.query +from ..misc import decanonicalize_hostname, get_distro, get_distro_version import cloud +import downburst +import openstack log = logging.getLogger(__name__) @@ -20,12 +20,12 @@ def create_if_vm(ctx, machine_name, _downburst=None): if _downburst: status_info = _downburst.status else: - status_info = get_status(machine_name) + status_info = teuthology.lock.query.get_status(machine_name) shortname = decanonicalize_hostname(machine_name) machine_type = status_info['machine_type'] os_type = get_distro(ctx) os_version = get_distro_version(ctx) - if not is_vm(status=status_info): + if not teuthology.lock.query.is_vm(status=status_info): return False if machine_type in cloud.get_types(): @@ -43,8 +43,9 @@ def create_if_vm(ctx, machine_name, _downburst=None): 'Usage of a custom downburst config has been deprecated.' ) - dbrst = _downburst or Downburst(name=machine_name, os_type=os_type, - os_version=os_version, status=status_info) + dbrst = _downburst or \ + downburst.Downburst(name=machine_name, os_type=os_type, + os_version=os_version, status=status_info) return dbrst.create() @@ -60,8 +61,8 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None, if _downburst: status_info = _downburst.status else: - status_info = get_status(machine_name) - if not status_info or not is_vm(status=status_info): + status_info = teuthology.lock.query.get_status(machine_name) + if not status_info or not teuthology.lock.query.is_vm(status=status_info): return True if user is not None and user != status_info['locked_by']: msg = "Tried to destroy {node} as {as_user} but it is locked " + \ @@ -79,11 +80,12 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None, machine_type = status_info.get('machine_type') shortname = decanonicalize_hostname(machine_name) if machine_type == 'openstack': - return ProvisionOpenStack().destroy(shortname) + return openstack.ProvisionOpenStack().destroy(shortname) elif machine_type in cloud.get_types(): return cloud.get_provisioner( machine_type, shortname, None, None).destroy() - dbrst = _downburst or Downburst(name=machine_name, os_type=None, - os_version=None, status=status_info) + dbrst = _downburst or \ + downburst.Downburst(name=machine_name, os_type=None, + os_version=None, status=status_info) return dbrst.destroy() diff --git a/teuthology/provision/downburst.py b/teuthology/provision/downburst.py index f5a0b15680..d9e300a900 100644 --- a/teuthology/provision/downburst.py +++ b/teuthology/provision/downburst.py @@ -1,3 +1,4 @@ +import json import logging import os import subprocess @@ -7,8 +8,7 @@ import yaml from ..config import config from ..contextutil import safe_while from ..misc import decanonicalize_hostname -from ..lock import get_status - +from teuthology.lock import query log = logging.getLogger(__name__) @@ -46,7 +46,7 @@ class Downburst(object): self.name = name self.os_type = os_type self.os_version = os_version - self.status = status or get_status(self.name) + self.status = status or query.get_status(self.name) self.config_path = None self.user_path = None self.user = user @@ -221,3 +221,40 @@ class Downburst(object): def __del__(self): self.remove_config() + + +def get_distro_from_downburst(): + """ + Return a table of valid distros. + + If downburst is in path use it. If either downburst is unavailable, + or if downburst is unable to produce a json list, then use a default + table. + """ + default_table = {u'rhel_minimal': [u'6.4', u'6.5'], + u'fedora': [u'17', u'18', u'19', u'20', u'22'], + u'centos': [u'6.3', u'6.4', u'6.5', u'7.0', + u'7.2'], + u'centos_minimal': [u'6.4', u'6.5'], + u'ubuntu': [u'8.04(hardy)', u'9.10(karmic)', + u'10.04(lucid)', u'10.10(maverick)', + u'11.04(natty)', u'11.10(oneiric)', + u'12.04(precise)', u'12.10(quantal)', + u'13.04(raring)', u'13.10(saucy)', + u'14.04(trusty)', u'utopic(utopic)', + u'16.04(xenial)'], + u'sles': [u'11-sp2'], + u'debian': [u'6.0', u'7.0', u'8.0']} + executable_cmd = downburst_executable() + if not executable_cmd: + log.warn("Downburst not found!") + log.info('Using default values for supported os_type/os_version') + return default_table + try: + output = subprocess.check_output([executable_cmd, 'list-json']) + downburst_data = json.loads(output) + return downburst_data + except (subprocess.CalledProcessError, OSError): + log.exception("Error calling downburst!") + log.info('Using default values for supported os_type/os_version') + return default_table \ No newline at end of file diff --git a/teuthology/provision/test/test_downburst.py b/teuthology/provision/test/test_downburst.py index 53d8e54c5f..cc947b2850 100644 --- a/teuthology/provision/test/test_downburst.py +++ b/teuthology/provision/test/test_downburst.py @@ -21,7 +21,7 @@ class TestDownburst(object): ctx = self.ctx status = self.status - dbrst = provision.Downburst( + dbrst = provision.downburst.Downburst( name, ctx.os_type, ctx.os_version, status) dbrst.executable = '/fake/path' dbrst.build_config = MagicMock(name='build_config') @@ -43,7 +43,7 @@ class TestDownburst(object): ctx = self.ctx status = self.status - dbrst = provision.Downburst( + dbrst = provision.downburst.Downburst( name, ctx.os_type, ctx.os_version, status) dbrst.destroy = MagicMock(name='destroy') dbrst.destroy.return_value = True @@ -59,7 +59,7 @@ class TestDownburst(object): status = self.status status['locked_by'] = 'user@a' - dbrst = provision.Downburst( + dbrst = provision.downburst.Downburst( name, ctx.os_type, ctx.os_version, status) dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError) @@ -73,7 +73,7 @@ class TestDownburst(object): status = self.status status['description'] = 'desc_a' - dbrst = provision.Downburst( + dbrst = provision.downburst.Downburst( name, ctx.os_type, ctx.os_version, status) dbrst.destroy = MagicMock(name='destroy') dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError) @@ -82,24 +82,24 @@ class TestDownburst(object): _downburst=dbrst) assert result is False - @patch('teuthology.provision_executable') + @patch('teuthology.provision.downburst.downburst_executable') def test_create_fails_without_executable(self, m_exec): name = self.name ctx = self.ctx status = self.status m_exec.return_value = '' - dbrst = provision.Downburst( + dbrst = provision.downburst.Downburst( name, ctx.os_type, ctx.os_version, status) result = dbrst.create() assert result is False - @patch('teuthology.provision_executable') + @patch('teuthology.provision.downburst.downburst_executable') def test_destroy_fails_without_executable(self, m_exec): name = self.name ctx = self.ctx status = self.status m_exec.return_value = '' - dbrst = provision.Downburst( + dbrst = provision.downburst.Downburst( name, ctx.os_type, ctx.os_version, status) result = dbrst.destroy() assert result is False diff --git a/teuthology/suite/test/test_util.py b/teuthology/suite/test/test_util.py index c587a205c6..bb13ab4549 100644 --- a/teuthology/suite/test/test_util.py +++ b/teuthology/suite/test/test_util.py @@ -87,17 +87,17 @@ class TestUtil(object): ) assert result == "some json" - @patch('teuthology.suite.util.lock') - def test_get_arch_fail(self, m_lock): - m_lock.list_locks.return_value = False + @patch('teuthology.lock.query') + def test_get_arch_fail(self, m_query): + m_query.list_locks.return_value = False util.get_arch('magna') - m_lock.list_locks.assert_called_with(machine_type="magna", count=1) + m_query.list_locks.assert_called_with(machine_type="magna", count=1) - @patch('teuthology.suite.util.lock') - def test_get_arch_success(self, m_lock): - m_lock.list_locks.return_value = [{"arch": "arch"}] + @patch('teuthology.lock.query') + def test_get_arch_success(self, m_query): + m_query.list_locks.return_value = [{"arch": "arch"}] result = util.get_arch('magna') - m_lock.list_locks.assert_called_with( + m_query.list_locks.assert_called_with( machine_type="magna", count=1 ) diff --git a/teuthology/suite/util.py b/teuthology/suite/util.py index 2538c12b08..0f7d205c2b 100644 --- a/teuthology/suite/util.py +++ b/teuthology/suite/util.py @@ -9,7 +9,8 @@ import sys from email.mime.text import MIMEText -from .. import lock +import teuthology.lock.query +import teuthology.lock.util from .. import repo_utils from ..config import config @@ -276,7 +277,7 @@ def get_arch(machine_type): :returns: A string or None """ - result = lock.list_locks(machine_type=machine_type, count=1) + result = teuthology.lock.query.list_locks(machine_type=machine_type, count=1) if not result: log.warn("No machines found with machine_type %s!", machine_type) else: diff --git a/teuthology/task/internal/__init__.py b/teuthology/task/internal/__init__.py index da88011f2d..c492b4ec53 100644 --- a/teuthology/task/internal/__init__.py +++ b/teuthology/task/internal/__init__.py @@ -11,7 +11,7 @@ import time import yaml import subprocess -from teuthology import lock +import teuthology.lock.ops from teuthology import misc from teuthology.packaging import get_builder_project from teuthology import report @@ -179,7 +179,7 @@ def push_inventory(ctx, config): def push(): for rem in ctx.cluster.remotes.keys(): info = rem.inventory_info - lock.update_inventory(info) + teuthology.lock.ops.update_inventory(info) try: push() except Exception: diff --git a/teuthology/task/internal/check_lock.py b/teuthology/task/internal/check_lock.py index 39a8c190da..ec01deaa82 100644 --- a/teuthology/task/internal/check_lock.py +++ b/teuthology/task/internal/check_lock.py @@ -1,6 +1,7 @@ import logging -from teuthology import lock +import teuthology.lock.query +import teuthology.lock.util from teuthology.config import config as teuth_config @@ -16,7 +17,7 @@ def check_lock(ctx, config, check_up=True): return log.info('Checking locks...') for machine in ctx.config['targets'].iterkeys(): - status = lock.get_status(machine) + status = teuthology.lock.query.get_status(machine) log.debug('machine status is %s', repr(status)) assert status is not None, \ 'could not read lock status for {name}'.format(name=machine) diff --git a/teuthology/task/internal/lock_machines.py b/teuthology/task/internal/lock_machines.py index 68210cd856..a44f9d2301 100644 --- a/teuthology/task/internal/lock_machines.py +++ b/teuthology/task/internal/lock_machines.py @@ -3,7 +3,10 @@ import logging import time import yaml -from teuthology import lock +import teuthology.lock.keys +import teuthology.lock.ops +import teuthology.lock.query +import teuthology.lock.util from teuthology import misc from teuthology import provision from teuthology import report @@ -44,8 +47,8 @@ def lock_machines(ctx, config): requested = total_requested while True: # get a candidate list of machines - machines = lock.list_locks(machine_type=machine_type, up=True, - locked=False, count=requested + reserved) + machines = teuthology.lock.query.list_locks(machine_type=machine_type, up=True, + locked=False, count=requested + reserved) if machines is None: if ctx.block: log.error('Error listing machines, trying again') @@ -71,9 +74,9 @@ def lock_machines(ctx, config): (reserved, requested, len(machines))) try: - newly_locked = lock.lock_many(ctx, requested, machine_type, - ctx.owner, ctx.archive, os_type, - os_version, arch) + newly_locked = teuthology.lock.ops.lock_many(ctx, requested, machine_type, + ctx.owner, ctx.archive, os_type, + os_version, arch) except Exception: # Lock failures should map to the 'dead' status instead of 'fail' set_status(ctx.summary, 'dead') @@ -91,7 +94,7 @@ def lock_machines(ctx, config): if len(all_locked) == total_requested: vmlist = [] for lmach in all_locked: - if lock.is_vm(lmach): + if teuthology.lock.query.is_vm(lmach): vmlist.append(lmach) if vmlist: log.info('Waiting for virtual machines to come up') @@ -112,11 +115,11 @@ def lock_machines(ctx, config): full_name = misc.canonicalize_hostname(guest) provision.destroy_if_vm(ctx, full_name) provision.create_if_vm(ctx, full_name) - if lock.do_update_keys(keys_dict): + if teuthology.lock.keys.do_update_keys(keys_dict): log.info("Error in virtual machine keys") newscandict = {} for dkey in all_locked.iterkeys(): - stats = lock.get_status(dkey) + stats = teuthology.lock.query.get_status(dkey) newscandict[dkey] = stats['ssh_pub_key'] ctx.config['targets'] = newscandict else: @@ -154,4 +157,4 @@ def lock_machines(ctx, config): if get_status(ctx.summary) == 'pass' or unlock_on_failure: log.info('Unlocking machines...') for machine in ctx.config['targets'].iterkeys(): - lock.unlock_one(ctx, machine, ctx.owner, ctx.archive) + teuthology.lock.ops.unlock_one(ctx, machine, ctx.owner, ctx.archive) diff --git a/teuthology/test/test_vps_os_vers_parameter_checking.py b/teuthology/test/test_vps_os_vers_parameter_checking.py index 9e9fa02190..3bcda1f87b 100644 --- a/teuthology/test/test_vps_os_vers_parameter_checking.py +++ b/teuthology/test/test_vps_os_vers_parameter_checking.py @@ -1,6 +1,7 @@ from mock import patch, Mock -from .. import lock +import teuthology.lock.util +from .. import provision class TestVpsOsVersionParamCheck(object): @@ -20,13 +21,13 @@ class TestVpsOsVersionParamCheck(object): self.fake_ctx.os_type = 'ubuntu' self.fake_ctx.os_version = 'precise' with patch.multiple( - lock.provision.downburst, + provision.downburst, downburst_executable=self.fake_downburst_executable, ): - check_value = lock.vps_version_or_type_valid( - self.fake_ctx.machine_type, - self.fake_ctx.os_type, - self.fake_ctx.os_version) + check_value = teuthology.lock.util.vps_version_or_type_valid( + self.fake_ctx.machine_type, + self.fake_ctx.os_type, + self.fake_ctx.os_version) assert check_value @@ -34,51 +35,50 @@ class TestVpsOsVersionParamCheck(object): self.fake_ctx.os_type = 'ubuntu' self.fake_ctx.os_version = '12.04' with patch.multiple( - lock.provision.downburst, + provision.downburst, downburst_executable=self.fake_downburst_executable, ): - check_value = lock.vps_version_or_type_valid( - self.fake_ctx.machine_type, - self.fake_ctx.os_type, - self.fake_ctx.os_version) + check_value = teuthology.lock.util.vps_version_or_type_valid( + self.fake_ctx.machine_type, + self.fake_ctx.os_type, + self.fake_ctx.os_version) assert check_value def test_mixup(self): self.fake_ctx.os_type = '6.5' self.fake_ctx.os_version = 'rhel' with patch.multiple( - lock.provision.downburst, + provision.downburst, downburst_executable=self.fake_downburst_executable, ): - check_value = lock.vps_version_or_type_valid( - self.fake_ctx.machine_type, - self.fake_ctx.os_type, - self.fake_ctx.os_version) + check_value = teuthology.lock.util.vps_version_or_type_valid( + self.fake_ctx.machine_type, + self.fake_ctx.os_type, + self.fake_ctx.os_version) assert not check_value def test_bad_type(self): self.fake_ctx.os_type = 'aardvark' self.fake_ctx.os_version = '6.5' with patch.multiple( - lock.provision.downburst, + provision.downburst, downburst_executable=self.fake_downburst_executable, ): - check_value = lock.vps_version_or_type_valid( - self.fake_ctx.machine_type, - self.fake_ctx.os_type, - self.fake_ctx.os_version) + check_value = teuthology.lock.util.vps_version_or_type_valid( + self.fake_ctx.machine_type, + self.fake_ctx.os_type, + self.fake_ctx.os_version) assert not check_value def test_bad_version(self): self.fake_ctx.os_type = 'rhel' self.fake_ctx.os_version = 'vampire_bat' with patch.multiple( - lock.provision.downburst, + provision.downburst, downburst_executable=self.fake_downburst_executable, ): - check_value = lock.vps_version_or_type_valid( - self.fake_ctx.machine_type, - self.fake_ctx.os_type, - self.fake_ctx.os_version) + check_value = teuthology.lock.util.vps_version_or_type_valid( + self.fake_ctx.machine_type, + self.fake_ctx.os_type, + self.fake_ctx.os_version) assert not check_value - -- 2.39.5