import logging
import sys
from teuthology.orchestra.remote import Remote
-from teuthology.lock import update_inventory
+from teuthology.lock.ops import update_inventory
+
paddles_url = 'http://paddles.example.com/nodes/'
machine_type = 'typica'
import sys
import teuthology.lock
+import teuthology.lock.cli
def _positive_int(string):
def main():
- sys.exit(teuthology.lock.main(parse_args(sys.argv[1:])))
+ sys.exit(teuthology.lock.cli.main(parse_args(sys.argv[1:])))
def parse_args(argv):
import teuthology
import teuthology.lock
+import teuthology.lock.ops
import teuthology.orchestra.remote
import logging
for rem_name in remotes:
remote = teuthology.orchestra.remote.Remote(rem_name)
inventory_info = remote.inventory_info
- teuthology.lock.update_inventory(inventory_info)
+ teuthology.lock.ops.update_inventory(inventory_info)
import sys
import teuthology.lock
+import teuthology.lock.cli
doc = """
usage: teuthology-updatekeys -h
def main():
args = docopt.docopt(doc)
- status = teuthology.lock.updatekeys(args)
+ status = teuthology.lock.cli.updatekeys(args)
sys.exit(status)
+++ /dev/null
-import argparse
-import datetime
-import json
-import logging
-import subprocess
-import yaml
-import re
-import collections
-import os
-import requests
-import urllib
-
-import teuthology
-from . import misc
-from . import provision
-from .config import config
-from .config import set_config_attr
-from .contextutil import safe_while
-
-log = logging.getLogger(__name__)
-
-
-def get_status(name):
- name = misc.canonicalize_hostname(name, user=None)
- uri = os.path.join(config.lock_server, 'nodes', name, '')
- response = requests.get(uri)
- success = response.ok
- if success:
- return response.json()
- log.warning(
- "Failed to query lock server for status of {name}".format(name=name))
- return None
-
-
-def is_vm(name=None, status=None):
- if status is None:
- if name is None:
- raise ValueError("Must provide either name or status, or both")
- name = misc.canonicalize_hostname(name)
- status = get_status(name)
- return status.get('is_vm', False)
-
-
-def get_distro_from_downburst():
- """
- Return a table of valid distros.
-
- If downburst is in path use it. If either downburst is unavailable,
- or if downburst is unable to produce a json list, then use a default
- table.
- """
- default_table = {u'rhel_minimal': [u'6.4', u'6.5'],
- u'fedora': [u'17', u'18', u'19', u'20', u'22'],
- u'centos': [u'6.3', u'6.4', u'6.5', u'7.0',
- u'7.2'],
- u'centos_minimal': [u'6.4', u'6.5'],
- u'ubuntu': [u'8.04(hardy)', u'9.10(karmic)',
- u'10.04(lucid)', u'10.10(maverick)',
- u'11.04(natty)', u'11.10(oneiric)',
- u'12.04(precise)', u'12.10(quantal)',
- u'13.04(raring)', u'13.10(saucy)',
- u'14.04(trusty)', u'utopic(utopic)',
- u'16.04(xenial)'],
- u'sles': [u'11-sp2'],
- u'debian': [u'6.0', u'7.0', u'8.0']}
- executable_cmd = provision.downburst.downburst_executable()
- if not executable_cmd:
- log.warn("Downburst not found!")
- log.info('Using default values for supported os_type/os_version')
- return default_table
- try:
- output = subprocess.check_output([executable_cmd, 'list-json'])
- downburst_data = json.loads(output)
- return downburst_data
- except (subprocess.CalledProcessError, OSError):
- log.exception("Error calling downburst!")
- log.info('Using default values for supported os_type/os_version')
- return default_table
-
-
-def vps_version_or_type_valid(machine_type, os_type, os_version):
- """
- Check os-type and os-version parameters when locking a vps.
- Os-type will always be set (defaults to ubuntu).
-
- In the case where downburst does not handle list-json (an older version
- of downburst, for instance), a message is printed and this checking
- is skipped (so that this code should behave as it did before this
- check was added).
- """
- if not machine_type == 'vps':
- return True
- if os_type is None or os_version is None:
- # we'll use the defaults provided by provision.create_if_vm
- # later on during provisioning
- return True
- valid_os_and_version = get_distro_from_downburst()
- if os_type not in valid_os_and_version:
- log.error("os-type '%s' is invalid. Try one of: %s",
- os_type,
- ', '.join(valid_os_and_version.keys()))
- return False
- if not validate_distro_version(os_version,
- valid_os_and_version[os_type]):
- log.error(
- "os-version '%s' is invalid for os-type '%s'. Try one of: %s",
- os_version,
- os_type,
- ', '.join(valid_os_and_version[os_type]))
- return False
- return True
-
-
-def validate_distro_version(version, supported_versions):
- """
- Return True if the version is valid. For Ubuntu, possible
- supported version values are of the form '12.04 (precise)' where
- either the number of the version name is acceptable.
- """
- if version in supported_versions:
- return True
- for parts in supported_versions:
- part = parts.split('(')
- if len(part) == 2:
- if version == part[0]:
- return True
- if version == part[1][0:len(part[1])-1]:
- return True
-
-
-def get_statuses(machines):
- if machines:
- statuses = []
- for machine in machines:
- machine = misc.canonicalize_hostname(machine)
- status = get_status(machine)
- if status:
- statuses.append(status)
- else:
- log.error("Lockserver doesn't know about machine: %s" %
- machine)
- else:
- statuses = list_locks()
- return statuses
-
-
-def json_matching_statuses(json_file_or_str, statuses):
- """
- Filter statuses by json dict in file or fragment; return list of
- matching statuses. json_file_or_str must be a file containing
- json or json in a string.
- """
- try:
- open(json_file_or_str, 'r')
- except IOError:
- query = json.loads(json_file_or_str)
- else:
- query = json.load(json_file_or_str)
-
- if not isinstance(query, dict):
- raise RuntimeError('--json-query must be a dict')
-
- return_statuses = list()
- for status in statuses:
- for k, v in query.iteritems():
- if not misc.is_in_dict(k, v, status):
- break
- else:
- return_statuses.append(status)
-
- return return_statuses
-
-
-def winnow(statuses, arg, status_key, func=None):
- """
- Call with a list of statuses, and the ctx.<key>
- 'arg' that you may want to filter by.
- If arg is not None, filter statuses by either:
-
- 1) func=None: filter by status[status_key] == arg
- remove any status that fails
-
- 2) func=<filter function that takes status>: remove any
- status for which func returns False
-
- Return the possibly-smaller set of statuses.
- """
-
- if arg is not None:
- if func:
- statuses = [_status for _status in statuses
- if func(_status)]
- else:
- statuses = [_status for _status in statuses
- if _status[status_key] == arg]
-
- return statuses
-
-
-def main(ctx):
- if ctx.verbose:
- teuthology.log.setLevel(logging.DEBUG)
-
- set_config_attr(ctx)
-
- ret = 0
- user = ctx.owner
- machines = [misc.canonicalize_hostname(m, user=False)
- for m in ctx.machines]
- machines_to_update = []
-
- if ctx.targets:
- try:
- with file(ctx.targets) as f:
- g = yaml.safe_load_all(f)
- for new in g:
- if 'targets' in new:
- for t in new['targets'].iterkeys():
- machines.append(t)
- except IOError as e:
- raise argparse.ArgumentTypeError(str(e))
-
- if ctx.f:
- assert ctx.lock or ctx.unlock, \
- '-f is only supported by --lock and --unlock'
- if machines:
- assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \
- or ctx.update or ctx.brief, \
- 'machines cannot be specified with that operation'
- else:
- if ctx.lock:
- log.error("--lock requires specific machines passed as arguments")
- else:
- # This condition might never be hit, but it's not clear.
- assert ctx.num_to_lock or ctx.list or ctx.list_targets or \
- ctx.summary or ctx.brief, \
- 'machines must be specified for that operation'
- if ctx.all:
- assert ctx.list or ctx.list_targets or ctx.brief, \
- '--all can only be used with --list, --list-targets, and --brief'
- assert ctx.owner is None, \
- '--all and --owner are mutually exclusive'
- assert not machines, \
- '--all and listing specific machines are incompatible'
- if ctx.num_to_lock:
- assert ctx.machine_type, \
- 'must specify machine type to lock'
-
- if ctx.brief or ctx.list or ctx.list_targets:
- assert ctx.desc is None, '--desc does nothing with --list/--brief'
-
- # we may need to update host keys for vms. Don't do it for
- # every vm; however, update any vms included in the list given
- # to the CLI (machines), or any owned by the specified owner or
- # invoking user if no machines are specified.
- vmachines = []
- statuses = get_statuses(machines)
- owner = ctx.owner or misc.get_user()
- for machine in statuses:
- if is_vm(status=machine) and machine['locked'] and \
- (machines or machine['locked_by'] == owner):
- vmachines.append(machine['name'])
- if vmachines:
- log.info("updating host keys for %s", ' '.join(sorted(vmachines)))
- do_update_keys(vmachines)
- # get statuses again to refresh any updated keys
- statuses = get_statuses(machines)
- if statuses:
- statuses = winnow(statuses, ctx.machine_type, 'machine_type')
- if not machines and ctx.owner is None and not ctx.all:
- ctx.owner = misc.get_user()
- statuses = winnow(statuses, ctx.owner, 'locked_by')
- statuses = winnow(statuses, ctx.status, 'up',
- lambda s: s['up'] == (ctx.status == 'up'))
- statuses = winnow(statuses, ctx.locked, 'locked',
- lambda s: s['locked'] == (ctx.locked == 'true'))
- statuses = winnow(statuses, ctx.desc, 'description')
- statuses = winnow(statuses, ctx.desc_pattern, 'description',
- lambda s: s['description'] and \
- ctx.desc_pattern in s['description'])
- if ctx.json_query:
- statuses = json_matching_statuses(ctx.json_query, statuses)
- statuses = winnow(statuses, ctx.os_type, 'os_type')
- statuses = winnow(statuses, ctx.os_version, 'os_version')
-
- # When listing, only show the vm_host's name, not every detail
- for s in statuses:
- if not is_vm(status=s):
- continue
- # with an OpenStack API, there is no host for a VM
- if s['vm_host'] is None:
- continue
- vm_host_name = s.get('vm_host', dict())['name']
- if vm_host_name:
- s['vm_host'] = vm_host_name
- if ctx.list:
- print json.dumps(statuses, indent=4)
-
- elif ctx.brief:
- for s in sorted(statuses, key=lambda s: s.get('name')):
- locked = "un" if s['locked'] == 0 else " "
- mo = re.match('\w+@(\w+?)\..*', s['name'])
- host = mo.group(1) if mo else s['name']
- print '{host} {locked}locked {owner} "{desc}"'.format(
- locked=locked, host=host,
- owner=s['locked_by'], desc=s['description'])
-
- else:
- frag = {'targets': {}}
- for f in statuses:
- frag['targets'][f['name']] = f['ssh_pub_key']
- print yaml.safe_dump(frag, default_flow_style=False)
- else:
- log.error('error retrieving lock statuses')
- ret = 1
-
- elif ctx.summary:
- do_summary(ctx)
- return 0
-
- elif ctx.lock:
- if not vps_version_or_type_valid(ctx.machine_type, ctx.os_type,
- ctx.os_version):
- log.error('Invalid os-type or version detected -- lock failed')
- return 1
- for machine in machines:
- if not lock_one(machine, user, ctx.desc):
- ret = 1
- if not ctx.f:
- return ret
- else:
- machines_to_update.append(machine)
- provision.create_if_vm(
- ctx,
- misc.canonicalize_hostname(machine),
- )
- elif ctx.unlock:
- if ctx.owner is None and user is None:
- user = misc.get_user()
- # If none of them are vpm, do them all in one shot
- if not filter(lambda x: is_vm(status=x), machines):
- res = unlock_many(machines, user)
- return 0 if res else 1
- for machine in machines:
- if not unlock_one(ctx, machine, user):
- ret = 1
- if not ctx.f:
- return ret
- else:
- machines_to_update.append(machine)
- elif ctx.num_to_lock:
- result = lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user,
- ctx.desc, ctx.os_type, ctx.os_version, ctx.arch)
- if not result:
- ret = 1
- else:
- machines_to_update = result.keys()
- if ctx.machine_type == 'vps':
- shortnames = ' '.join(
- [misc.decanonicalize_hostname(name) for name in
- result.keys()]
- )
- if len(result) < ctx.num_to_lock:
- log.error("Locking failed.")
- for machine in result:
- unlock_one(ctx, machine, user)
- ret = 1
- else:
- log.info("Successfully Locked:\n%s\n" % shortnames)
- log.info(
- "Unable to display keys at this time (virtual " +
- "machines are booting).")
- log.info(
- "Please run teuthology-lock --list-targets %s once " +
- "these machines come up.",
- shortnames)
- else:
- print yaml.safe_dump(
- dict(targets=result),
- default_flow_style=False)
- elif ctx.update:
- assert ctx.desc is not None or ctx.status is not None, \
- 'you must specify description or status to update'
- assert ctx.owner is None, 'only description and status may be updated'
- machines_to_update = machines
-
- if ctx.desc is not None or ctx.status is not None:
- for machine in machines_to_update:
- update_lock(machine, ctx.desc, ctx.status)
-
- return ret
-
-
-def lock_many_openstack(ctx, num, machine_type, user=None, description=None,
- arch=None):
- os_type = provision.get_distro(ctx)
- os_version = provision.get_distro_version(ctx)
- if hasattr(ctx, 'config'):
- resources_hint = ctx.config.get('openstack')
- else:
- resources_hint = None
- machines = provision.ProvisionOpenStack().create(
- num, os_type, os_version, arch, resources_hint)
- result = {}
- for machine in machines:
- lock_one(machine, user, description)
- result[machine] = None # we do not collect ssh host keys yet
- return result
-
-def lock_many(ctx, num, machine_type, user=None, description=None,
- os_type=None, os_version=None, arch=None):
- if user is None:
- user = misc.get_user()
-
- if not vps_version_or_type_valid(ctx.machine_type, os_type, os_version):
- log.error('Invalid os-type or version detected -- lock failed')
- return
-
- # In the for loop below we can safely query for all bare-metal machine_type
- # values at once. So, if we're being asked for 'plana,mira,burnupi', do it
- # all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query
- # for 'plana,mira,burnupi' and one for 'vps'
- machine_types_list = misc.get_multi_machine_types(machine_type)
- if machine_types_list == ['vps']:
- machine_types = machine_types_list
- elif machine_types_list == ['openstack']:
- return lock_many_openstack(ctx, num, machine_type,
- user=user,
- description=description,
- arch=arch)
- elif 'vps' in machine_types_list:
- machine_types_non_vps = list(machine_types_list)
- machine_types_non_vps.remove('vps')
- machine_types_non_vps = '|'.join(machine_types_non_vps)
- machine_types = [machine_types_non_vps, 'vps']
- else:
- machine_types_str = '|'.join(machine_types_list)
- machine_types = [machine_types_str, ]
-
- for machine_type in machine_types:
- uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '')
- data = dict(
- locked_by=user,
- count=num,
- machine_type=machine_type,
- description=description,
- )
- # Only query for os_type/os_version if non-vps and non-libcloud, since
- # in that case we just create them.
- if machine_type not in ['vps'] + provision.cloud.get_types():
- if os_type:
- data['os_type'] = os_type
- if os_version:
- data['os_version'] = os_version
- if arch:
- data['arch'] = arch
- log.debug("lock_many request: %s", repr(data))
- response = requests.post(
- uri,
- data=json.dumps(data),
- headers={'content-type': 'application/json'},
- )
- if response.ok:
- machines = {misc.canonicalize_hostname(machine['name']):
- machine['ssh_pub_key'] for machine in response.json()}
- log.debug('locked {machines}'.format(
- machines=', '.join(machines.keys())))
- if machine_type in ['vps'] + provision.cloud.get_types():
- ok_machs = {}
- for machine in machines:
- if provision.create_if_vm(ctx, machine):
- ok_machs[machine] = machines[machine]
- else:
- log.error('Unable to create virtual machine: %s',
- machine)
- unlock_one(ctx, machine, user)
- return ok_machs
- if machine_type in provision.cloud.get_types():
- machines = do_update_keys(machines.keys())
- return machines
- elif response.status_code == 503:
- log.error('Insufficient nodes available to lock %d %s nodes.',
- num, machine_type)
- log.error(response.text)
- else:
- log.error('Could not lock %d %s nodes, reason: unknown.',
- num, machine_type)
- return []
-
-
-def lock_one(name, user=None, description=None):
- name = misc.canonicalize_hostname(name, user=None)
- if user is None:
- user = misc.get_user()
- request = dict(name=name, locked=True, locked_by=user,
- description=description)
- uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
- response = requests.put(uri, json.dumps(request))
- success = response.ok
- if success:
- log.debug('locked %s as %s', name, user)
- else:
- try:
- reason = response.json().get('message')
- except ValueError:
- reason = str(response.status_code)
- log.error('failed to lock {node}. reason: {reason}'.format(
- node=name, reason=reason))
- return response
-
-
-def unlock_many(names, user):
- fixed_names = [misc.canonicalize_hostname(name, user=None) for name in
- names]
- names = fixed_names
- uri = os.path.join(config.lock_server, 'nodes', 'unlock_many', '')
- data = dict(
- locked_by=user,
- names=names,
- )
- response = requests.post(
- uri,
- data=json.dumps(data),
- headers={'content-type': 'application/json'},
- )
- if response.ok:
- log.debug("Unlocked: %s", ', '.join(names))
- else:
- log.error("Failed to unlock: %s", ', '.join(names))
- return response.ok
-
-
-def unlock_one(ctx, name, user, description=None):
- name = misc.canonicalize_hostname(name, user=None)
- if not provision.destroy_if_vm(ctx, name, user, description):
- log.error('destroy failed for %s', name)
- request = dict(name=name, locked=False, locked_by=user,
- description=description)
- uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
- with safe_while(
- sleep=1, increment=0.5, action="unlock %s" % name) as proceed:
- while proceed():
- try:
- response = requests.put(uri, json.dumps(request))
- break
- # Work around https://github.com/kennethreitz/requests/issues/2364
- except requests.ConnectionError as e:
- log.warn("Saw %s while unlocking; retrying...", str(e))
- success = response.ok
- if success:
- log.info('unlocked %s', name)
- else:
- try:
- reason = response.json().get('message')
- except ValueError:
- reason = str(response.status_code)
- log.error('failed to unlock {node}. reason: {reason}'.format(
- node=name, reason=reason))
- return success
-
-def locked_since_seconds(node):
- now = datetime.datetime.now()
- since = datetime.datetime.strptime(
- node['locked_since'], '%Y-%m-%d %H:%M:%S.%f')
- return (now - since).total_seconds()
-
-def list_locks(keyed_by_name=False, **kwargs):
- uri = os.path.join(config.lock_server, 'nodes', '')
- for key, value in kwargs.iteritems():
- if kwargs[key] is False:
- kwargs[key] = '0'
- if kwargs[key] is True:
- kwargs[key] = '1'
- if kwargs:
- if 'machine_type' in kwargs:
- kwargs['machine_type'] = kwargs['machine_type'].replace(',','|')
- uri += '?' + urllib.urlencode(kwargs)
- try:
- response = requests.get(uri)
- except requests.ConnectionError:
- success = False
- log.exception("Could not contact lock server: %s", config.lock_server)
- else:
- success = response.ok
- if success:
- if not keyed_by_name:
- return response.json()
- else:
- return {node['name']: node
- for node in response.json()}
- return dict()
-
-
-def find_stale_locks(owner=None):
- """
- Return a list of node dicts corresponding to nodes that were locked to run
- a job, but the job is no longer running. The purpose of this is to enable
- us to nuke nodes that were left locked due to e.g. infrastructure failures
- and return them to the pool.
-
- :param owner: If non-None, return nodes locked by owner. Default is None.
- """
- def might_be_stale(node_dict):
- """
- Answer the question: "might this be a stale lock?"
-
- The answer is yes if:
- It is locked
- It has a non-null description containing multiple '/' characters
-
- ... because we really want "nodes that were locked for a particular job
- and are still locked" and the above is currently the best way to guess.
- """
- desc = node_dict['description']
- if (node_dict['locked'] is True and
- desc is not None and desc.startswith('/') and
- desc.count('/') > 1):
- return True
- return False
-
- # Which nodes are locked for jobs?
- nodes = list_locks(locked=True)
- if owner is not None:
- nodes = [node for node in nodes if node['locked_by'] == owner]
- nodes = filter(might_be_stale, nodes)
-
- def node_job_is_active(node, cache):
- """
- Is this node's job active (e.g. running or waiting)?
-
- :param node: The node dict as returned from the lock server
- :param cache: A set() used for caching results
- :returns: True or False
- """
- description = node['description']
- if description in cache:
- return True
- (name, job_id) = description.split('/')[-2:]
- url = os.path.join(config.results_server, 'runs', name, 'jobs', job_id,
- '')
- resp = requests.get(url)
- job_info = resp.json()
- if job_info['status'] in ('running', 'waiting'):
- cache.add(description)
- return True
- return False
-
- result = list()
- # Here we build the list of of nodes that are locked, for a job (as opposed
- # to being locked manually for random monkeying), where the job is not
- # running
- active_jobs = set()
- for node in nodes:
- if node_job_is_active(node, active_jobs):
- continue
- result.append(node)
- return result
-
-
-def update_lock(name, description=None, status=None, ssh_pub_key=None):
- name = misc.canonicalize_hostname(name, user=None)
- updated = {}
- if description is not None:
- updated['description'] = description
- if status is not None:
- updated['up'] = (status == 'up')
- if ssh_pub_key is not None:
- updated['ssh_pub_key'] = ssh_pub_key
-
- if updated:
- uri = os.path.join(config.lock_server, 'nodes', name, '')
- response = requests.put(
- uri,
- json.dumps(updated))
- return response.ok
- return True
-
-
-def update_inventory(node_dict):
- """
- Like update_lock(), but takes a dict and doesn't try to do anything smart
- by itself
- """
- name = node_dict.get('name')
- if not name:
- raise ValueError("must specify name")
- if not config.lock_server:
- return
- uri = os.path.join(config.lock_server, 'nodes', name, '')
- log.info("Updating %s on lock server", name)
- response = requests.put(
- uri,
- json.dumps(node_dict),
- headers={'content-type': 'application/json'},
- )
- if response.status_code == 404:
- log.info("Creating new node %s on lock server", name)
- uri = os.path.join(config.lock_server, 'nodes', '')
- response = requests.post(
- uri,
- json.dumps(node_dict),
- headers={'content-type': 'application/json'},
- )
- if not response.ok:
- log.error("Node update/creation failed for %s: %s",
- name, response.text)
- return response.ok
-
-
-def updatekeys(args):
- loglevel = logging.DEBUG if args['--verbose'] else logging.INFO
- logging.basicConfig(
- level=loglevel,
- )
- all_ = args['--all']
- if all_:
- machines = []
- elif args['<machine>']:
- machines = [misc.canonicalize_hostname(m, user=None)
- for m in args['<machine>']]
- elif args['--targets']:
- targets = args['--targets']
- with file(targets) as f:
- docs = yaml.safe_load_all(f)
- for doc in docs:
- machines = [n for n in doc.get('targets', dict()).iterkeys()]
-
- return do_update_keys(machines, all_)
-
-
-def do_update_keys(machines, all_=False):
- reference = list_locks(keyed_by_name=True)
- if all_:
- machines = reference.keys()
- keys_dict = misc.ssh_keyscan(machines)
- return push_new_keys(keys_dict, reference)
-
-
-def push_new_keys(keys_dict, reference):
- ret = 0
- for hostname, pubkey in keys_dict.iteritems():
- log.info('Checking %s', hostname)
- if reference[hostname]['ssh_pub_key'] != pubkey:
- log.info('New key found. Updating...')
- if not update_lock(hostname, ssh_pub_key=pubkey):
- log.error('failed to update %s!', hostname)
- ret = 1
- return ret
-
-
-def do_summary(ctx):
- lockd = collections.defaultdict(lambda: [0, 0, 'unknown'])
- if ctx.machine_type:
- locks = list_locks(machine_type=ctx.machine_type)
- else:
- locks = list_locks()
- for l in locks:
- who = l['locked_by'] if l['locked'] == 1 \
- else '(free)', l['machine_type']
- lockd[who][0] += 1
- lockd[who][1] += 1 if l['up'] else 0
- lockd[who][2] = l['machine_type']
-
- locks = sorted([p for p in lockd.iteritems()
- ], key=lambda sort: (sort[1][2], sort[1][0]))
- total_count, total_up = 0, 0
- print "TYPE COUNT UP OWNER"
-
- for (owner, (count, upcount, machinetype)) in locks:
- # if machinetype == spectype:
- print "{machinetype:8s} {count:3d} {up:3d} {owner}".format(
- count=count, up=upcount, owner=owner[0],
- machinetype=machinetype)
- total_count += count
- total_up += upcount
-
- print " --- ---"
- print "{cnt:12d} {up:3d}".format(cnt=total_count, up=total_up)
--- /dev/null
+from teuthology.lock import cli, keys, ops, query, util # noqa
\ No newline at end of file
--- /dev/null
+import argparse
+import collections
+import json
+import logging
+import re
+
+import yaml
+
+import teuthology
+import teuthology.provision
+from teuthology import misc
+from teuthology.config import set_config_attr
+
+from teuthology.lock import (
+ keys,
+ ops,
+ util,
+ query,
+)
+
+
+log = logging.getLogger(__name__)
+
+
+def main(ctx):
+ if ctx.verbose:
+ teuthology.log.setLevel(logging.DEBUG)
+
+ set_config_attr(ctx)
+
+ ret = 0
+ user = ctx.owner
+ machines = [misc.canonicalize_hostname(m, user=False)
+ for m in ctx.machines]
+ machines_to_update = []
+
+ if ctx.targets:
+ try:
+ with file(ctx.targets) as f:
+ g = yaml.safe_load_all(f)
+ for new in g:
+ if 'targets' in new:
+ for t in new['targets'].iterkeys():
+ machines.append(t)
+ except IOError as e:
+ raise argparse.ArgumentTypeError(str(e))
+
+ if ctx.f:
+ assert ctx.lock or ctx.unlock, \
+ '-f is only supported by --lock and --unlock'
+ if machines:
+ assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \
+ or ctx.update or ctx.brief, \
+ 'machines cannot be specified with that operation'
+ else:
+ if ctx.lock:
+ log.error("--lock requires specific machines passed as arguments")
+ else:
+ # This condition might never be hit, but it's not clear.
+ assert ctx.num_to_lock or ctx.list or ctx.list_targets or \
+ ctx.summary or ctx.brief, \
+ 'machines must be specified for that operation'
+ if ctx.all:
+ assert ctx.list or ctx.list_targets or ctx.brief, \
+ '--all can only be used with --list, --list-targets, and --brief'
+ assert ctx.owner is None, \
+ '--all and --owner are mutually exclusive'
+ assert not machines, \
+ '--all and listing specific machines are incompatible'
+ if ctx.num_to_lock:
+ assert ctx.machine_type, \
+ 'must specify machine type to lock'
+
+ if ctx.brief or ctx.list or ctx.list_targets:
+ assert ctx.desc is None, '--desc does nothing with --list/--brief'
+
+ # we may need to update host keys for vms. Don't do it for
+ # every vm; however, update any vms included in the list given
+ # to the CLI (machines), or any owned by the specified owner or
+ # invoking user if no machines are specified.
+ vmachines = []
+ statuses = query.get_statuses(machines)
+ owner = ctx.owner or misc.get_user()
+ for machine in statuses:
+ if query.is_vm(status=machine) and machine['locked'] and \
+ (machines or machine['locked_by'] == owner):
+ vmachines.append(machine['name'])
+ if vmachines:
+ log.info("updating host keys for %s", ' '.join(sorted(vmachines)))
+ keys.do_update_keys(vmachines)
+ # get statuses again to refresh any updated keys
+ statuses = query.get_statuses(machines)
+ if statuses:
+ statuses = util.winnow(statuses, ctx.machine_type, 'machine_type')
+ if not machines and ctx.owner is None and not ctx.all:
+ ctx.owner = misc.get_user()
+ statuses = util.winnow(statuses, ctx.owner, 'locked_by')
+ statuses = util.winnow(statuses, ctx.status, 'up',
+ lambda s: s['up'] == (ctx.status == 'up'))
+ statuses = util.winnow(statuses, ctx.locked, 'locked',
+ lambda s: s['locked'] == (ctx.locked == 'true'))
+ statuses = util.winnow(statuses, ctx.desc, 'description')
+ statuses = util.winnow(statuses, ctx.desc_pattern, 'description',
+ lambda s: s['description'] and \
+ ctx.desc_pattern in s['description'])
+ if ctx.json_query:
+ statuses = util.json_matching_statuses(ctx.json_query, statuses)
+ statuses = util.winnow(statuses, ctx.os_type, 'os_type')
+ statuses = util.winnow(statuses, ctx.os_version, 'os_version')
+
+ # When listing, only show the vm_host's name, not every detail
+ for s in statuses:
+ if not query.is_vm(status=s):
+ continue
+ # with an OpenStack API, there is no host for a VM
+ if s['vm_host'] is None:
+ continue
+ vm_host_name = s.get('vm_host', dict())['name']
+ if vm_host_name:
+ s['vm_host'] = vm_host_name
+ if ctx.list:
+ print json.dumps(statuses, indent=4)
+
+ elif ctx.brief:
+ for s in sorted(statuses, key=lambda s: s.get('name')):
+ locked = "un" if s['locked'] == 0 else " "
+ mo = re.match('\w+@(\w+?)\..*', s['name'])
+ host = mo.group(1) if mo else s['name']
+ print '{host} {locked}locked {owner} "{desc}"'.format(
+ locked=locked, host=host,
+ owner=s['locked_by'], desc=s['description'])
+
+ else:
+ frag = {'targets': {}}
+ for f in statuses:
+ frag['targets'][f['name']] = f['ssh_pub_key']
+ print yaml.safe_dump(frag, default_flow_style=False)
+ else:
+ log.error('error retrieving lock statuses')
+ ret = 1
+
+ elif ctx.summary:
+ do_summary(ctx)
+ return 0
+
+ elif ctx.lock:
+ if not util.vps_version_or_type_valid(ctx.machine_type, ctx.os_type,
+ ctx.os_version):
+ log.error('Invalid os-type or version detected -- lock failed')
+ return 1
+ for machine in machines:
+ if not ops.lock_one(machine, user, ctx.desc):
+ ret = 1
+ if not ctx.f:
+ return ret
+ else:
+ machines_to_update.append(machine)
+ teuthology.provision.create_if_vm(
+ ctx,
+ misc.canonicalize_hostname(machine),
+ )
+ elif ctx.unlock:
+ if ctx.owner is None and user is None:
+ user = misc.get_user()
+ # If none of them are vpm, do them all in one shot
+ if not filter(query.is_vm, machines):
+ res = ops.unlock_many(machines, user)
+ return 0 if res else 1
+ for machine in machines:
+ if not ops.unlock_one(ctx, machine, user):
+ ret = 1
+ if not ctx.f:
+ return ret
+ else:
+ machines_to_update.append(machine)
+ elif ctx.num_to_lock:
+ result = ops.lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user,
+ ctx.desc, ctx.os_type, ctx.os_version, ctx.arch)
+ if not result:
+ ret = 1
+ else:
+ machines_to_update = result.keys()
+ if ctx.machine_type == 'vps':
+ shortnames = ' '.join(
+ [misc.decanonicalize_hostname(name) for name in
+ result.keys()]
+ )
+ if len(result) < ctx.num_to_lock:
+ log.error("Locking failed.")
+ for machine in result:
+ ops.unlock_one(ctx, machine, user)
+ ret = 1
+ else:
+ log.info("Successfully Locked:\n%s\n" % shortnames)
+ log.info(
+ "Unable to display keys at this time (virtual " +
+ "machines are booting).")
+ log.info(
+ "Please run teuthology-lock --list-targets %s once " +
+ "these machines come up.",
+ shortnames)
+ else:
+ print yaml.safe_dump(
+ dict(targets=result),
+ default_flow_style=False)
+ elif ctx.update:
+ assert ctx.desc is not None or ctx.status is not None, \
+ 'you must specify description or status to update'
+ assert ctx.owner is None, 'only description and status may be updated'
+ machines_to_update = machines
+
+ if ctx.desc is not None or ctx.status is not None:
+ for machine in machines_to_update:
+ ops.update_lock(machine, ctx.desc, ctx.status)
+
+ return ret
+
+def do_summary(ctx):
+ lockd = collections.defaultdict(lambda: [0, 0, 'unknown'])
+ if ctx.machine_type:
+ locks = query.list_locks(machine_type=ctx.machine_type)
+ else:
+ locks = query.list_locks()
+ for l in locks:
+ who = l['locked_by'] if l['locked'] == 1 \
+ else '(free)', l['machine_type']
+ lockd[who][0] += 1
+ lockd[who][1] += 1 if l['up'] else 0
+ lockd[who][2] = l['machine_type']
+
+ locks = sorted([p for p in lockd.iteritems()
+ ], key=lambda sort: (sort[1][2], sort[1][0]))
+ total_count, total_up = 0, 0
+ print "TYPE COUNT UP OWNER"
+
+ for (owner, (count, upcount, machinetype)) in locks:
+ # if machinetype == spectype:
+ print "{machinetype:8s} {count:3d} {up:3d} {owner}".format(
+ count=count, up=upcount, owner=owner[0],
+ machinetype=machinetype)
+ total_count += count
+ total_up += upcount
+
+ print " --- ---"
+ print "{cnt:12d} {up:3d}".format(cnt=total_count, up=total_up)
+
+
+def updatekeys(args):
+ loglevel = logging.DEBUG if args['--verbose'] else logging.INFO
+ logging.basicConfig(
+ level=loglevel,
+ )
+ all_ = args['--all']
+ if all_:
+ machines = []
+ elif args['<machine>']:
+ machines = [misc.canonicalize_hostname(m, user=None)
+ for m in args['<machine>']]
+ elif args['--targets']:
+ targets = args['--targets']
+ with file(targets) as f:
+ docs = yaml.safe_load_all(f)
+ for doc in docs:
+ machines = [n for n in doc.get('targets', dict()).iterkeys()]
+
+ return keys.do_update_keys(machines, all_)
\ No newline at end of file
--- /dev/null
+import logging
+
+from teuthology import misc
+
+from . import ops, query
+
+log = logging.getLogger(__name__)
+
+
+def do_update_keys(machines, all_=False):
+ reference = query.list_locks(keyed_by_name=True)
+ if all_:
+ machines = reference.keys()
+ keys_dict = misc.ssh_keyscan(machines)
+ return push_new_keys(keys_dict, reference)
+
+
+def push_new_keys(keys_dict, reference):
+ ret = 0
+ for hostname, pubkey in keys_dict.iteritems():
+ log.info('Checking %s', hostname)
+ if reference[hostname]['ssh_pub_key'] != pubkey:
+ log.info('New key found. Updating...')
+ if not ops.update_lock(hostname, ssh_pub_key=pubkey):
+ log.error('failed to update %s!', hostname)
+ ret = 1
+ return ret
\ No newline at end of file
--- /dev/null
+import logging
+import json
+import os
+
+import requests
+
+import teuthology.provision
+from teuthology import misc
+from teuthology.config import config
+from teuthology.contextutil import safe_while
+
+import util
+import keys
+
+log = logging.getLogger(__name__)
+
+
+def lock_many_openstack(ctx, num, machine_type, user=None, description=None,
+ arch=None):
+ os_type = teuthology.provision.get_distro(ctx)
+ os_version = teuthology.provision.get_distro_version(ctx)
+ if hasattr(ctx, 'config'):
+ resources_hint = ctx.config.get('openstack')
+ else:
+ resources_hint = None
+ machines = teuthology.provision.openstack.ProvisionOpenStack().create(
+ num, os_type, os_version, arch, resources_hint)
+ result = {}
+ for machine in machines:
+ lock_one(machine, user, description)
+ result[machine] = None # we do not collect ssh host keys yet
+ return result
+
+
+def lock_many(ctx, num, machine_type, user=None, description=None,
+ os_type=None, os_version=None, arch=None):
+ if user is None:
+ user = misc.get_user()
+
+ if not util.vps_version_or_type_valid(
+ ctx.machine_type,
+ os_type,
+ os_version
+ ):
+ log.error('Invalid os-type or version detected -- lock failed')
+ return
+
+ # In the for loop below we can safely query for all bare-metal machine_type
+ # values at once. So, if we're being asked for 'plana,mira,burnupi', do it
+ # all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query
+ # for 'plana,mira,burnupi' and one for 'vps'
+ machine_types_list = misc.get_multi_machine_types(machine_type)
+ if machine_types_list == ['vps']:
+ machine_types = machine_types_list
+ elif machine_types_list == ['openstack']:
+ return lock_many_openstack(ctx, num, machine_type,
+ user=user,
+ description=description,
+ arch=arch)
+ elif 'vps' in machine_types_list:
+ machine_types_non_vps = list(machine_types_list)
+ machine_types_non_vps.remove('vps')
+ machine_types_non_vps = '|'.join(machine_types_non_vps)
+ machine_types = [machine_types_non_vps, 'vps']
+ else:
+ machine_types_str = '|'.join(machine_types_list)
+ machine_types = [machine_types_str, ]
+
+ for machine_type in machine_types:
+ uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '')
+ data = dict(
+ locked_by=user,
+ count=num,
+ machine_type=machine_type,
+ description=description,
+ )
+ # Only query for os_type/os_version if non-vps and non-libcloud, since
+ # in that case we just create them.
+ vm_types = ['vps'] + teuthology.provision.cloud.get_types()
+ if machine_type not in vm_types:
+ if os_type:
+ data['os_type'] = os_type
+ if os_version:
+ data['os_version'] = os_version
+ if arch:
+ data['arch'] = arch
+ log.debug("lock_many request: %s", repr(data))
+ response = requests.post(
+ uri,
+ data=json.dumps(data),
+ headers={'content-type': 'application/json'},
+ )
+ if response.ok:
+ machines = {misc.canonicalize_hostname(machine['name']):
+ machine['ssh_pub_key'] for machine in response.json()}
+ log.debug('locked {machines}'.format(
+ machines=', '.join(machines.keys())))
+ if machine_type in vm_types:
+ ok_machs = {}
+ for machine in machines:
+ if teuthology.provision.create_if_vm(ctx, machine):
+ ok_machs[machine] = machines[machine]
+ else:
+ log.error('Unable to create virtual machine: %s',
+ machine)
+ unlock_one(ctx, machine, user)
+ return ok_machs
+ return machines
+ elif response.status_code == 503:
+ log.error('Insufficient nodes available to lock %d %s nodes.',
+ num, machine_type)
+ log.error(response.text)
+ else:
+ log.error('Could not lock %d %s nodes, reason: unknown.',
+ num, machine_type)
+ return []
+
+
+def lock_one(name, user=None, description=None):
+ name = misc.canonicalize_hostname(name, user=None)
+ if user is None:
+ user = misc.get_user()
+ request = dict(name=name, locked=True, locked_by=user,
+ description=description)
+ uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
+ response = requests.put(uri, json.dumps(request))
+ success = response.ok
+ if success:
+ log.debug('locked %s as %s', name, user)
+ else:
+ try:
+ reason = response.json().get('message')
+ except ValueError:
+ reason = str(response.status_code)
+ log.error('failed to lock {node}. reason: {reason}'.format(
+ node=name, reason=reason))
+ return response
+
+
+def unlock_many(names, user):
+ fixed_names = [misc.canonicalize_hostname(name, user=None) for name in
+ names]
+ names = fixed_names
+ uri = os.path.join(config.lock_server, 'nodes', 'unlock_many', '')
+ data = dict(
+ locked_by=user,
+ names=names,
+ )
+ response = requests.post(
+ uri,
+ data=json.dumps(data),
+ headers={'content-type': 'application/json'},
+ )
+ if response.ok:
+ log.debug("Unlocked: %s", ', '.join(names))
+ else:
+ log.error("Failed to unlock: %s", ', '.join(names))
+ return response.ok
+
+
+def unlock_one(ctx, name, user, description=None):
+ name = misc.canonicalize_hostname(name, user=None)
+ if not teuthology.provision.destroy_if_vm(ctx, name, user, description):
+ log.error('destroy failed for %s', name)
+ request = dict(name=name, locked=False, locked_by=user,
+ description=description)
+ uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
+ with safe_while(
+ sleep=1, increment=0.5, action="unlock %s" % name) as proceed:
+ while proceed():
+ try:
+ response = requests.put(uri, json.dumps(request))
+ break
+ # Work around https://github.com/kennethreitz/requests/issues/2364
+ except requests.ConnectionError as e:
+ log.warn("Saw %s while unlocking; retrying...", str(e))
+ success = response.ok
+ if success:
+ log.info('unlocked %s', name)
+ else:
+ try:
+ reason = response.json().get('message')
+ except ValueError:
+ reason = str(response.status_code)
+ log.error('failed to unlock {node}. reason: {reason}'.format(
+ node=name, reason=reason))
+ return success
+
+
+def update_lock(name, description=None, status=None, ssh_pub_key=None):
+ name = misc.canonicalize_hostname(name, user=None)
+ updated = {}
+ if description is not None:
+ updated['description'] = description
+ if status is not None:
+ updated['up'] = (status == 'up')
+ if ssh_pub_key is not None:
+ updated['ssh_pub_key'] = ssh_pub_key
+
+ if updated:
+ uri = os.path.join(config.lock_server, 'nodes', name, '')
+ response = requests.put(
+ uri,
+ json.dumps(updated))
+ return response.ok
+ return True
+
+
+def update_inventory(node_dict):
+ """
+ Like update_lock(), but takes a dict and doesn't try to do anything smart
+ by itself
+ """
+ name = node_dict.get('name')
+ if not name:
+ raise ValueError("must specify name")
+ if not config.lock_server:
+ return
+ uri = os.path.join(config.lock_server, 'nodes', name, '')
+ log.info("Updating %s on lock server", name)
+ response = requests.put(
+ uri,
+ json.dumps(node_dict),
+ headers={'content-type': 'application/json'},
+ )
+ if response.status_code == 404:
+ log.info("Creating new node %s on lock server", name)
+ uri = os.path.join(config.lock_server, 'nodes', '')
+ response = requests.post(
+ uri,
+ json.dumps(node_dict),
+ headers={'content-type': 'application/json'},
+ )
+ if not response.ok:
+ log.error("Node update/creation failed for %s: %s",
+ name, response.text)
+ return response.ok
--- /dev/null
+import logging
+import os
+import urllib
+
+import requests
+
+from teuthology import misc
+from teuthology.config import config
+
+
+log = logging.getLogger(__name__)
+
+
+def get_status(name):
+ name = misc.canonicalize_hostname(name, user=None)
+ uri = os.path.join(config.lock_server, 'nodes', name, '')
+ response = requests.get(uri)
+ success = response.ok
+ if success:
+ return response.json()
+ log.warning(
+ "Failed to query lock server for status of {name}".format(name=name))
+ return None
+
+
+def get_statuses(machines):
+ if machines:
+ statuses = []
+ for machine in machines:
+ machine = misc.canonicalize_hostname(machine)
+ status = get_status(machine)
+ if status:
+ statuses.append(status)
+ else:
+ log.error("Lockserver doesn't know about machine: %s" %
+ machine)
+ else:
+ statuses = list_locks()
+ return statuses
+
+
+def is_vm(name=None, status=None):
+ if status is None:
+ if name is None:
+ raise ValueError("Must provide either name or status, or both")
+ name = misc.canonicalize_hostname(name)
+ status = get_status(name)
+ return status.get('is_vm', False)
+
+
+def list_locks(keyed_by_name=False, **kwargs):
+ uri = os.path.join(config.lock_server, 'nodes', '')
+ for key, value in kwargs.iteritems():
+ if kwargs[key] is False:
+ kwargs[key] = '0'
+ if kwargs[key] is True:
+ kwargs[key] = '1'
+ if kwargs:
+ if 'machine_type' in kwargs:
+ kwargs['machine_type'] = kwargs['machine_type'].replace(',','|')
+ uri += '?' + urllib.urlencode(kwargs)
+ try:
+ response = requests.get(uri)
+ except requests.ConnectionError:
+ success = False
+ log.exception("Could not contact lock server: %s", config.lock_server)
+ else:
+ success = response.ok
+ if success:
+ if not keyed_by_name:
+ return response.json()
+ else:
+ return {node['name']: node
+ for node in response.json()}
+ return dict()
+
+
+def find_stale_locks(owner=None):
+ """
+ Return a list of node dicts corresponding to nodes that were locked to run
+ a job, but the job is no longer running. The purpose of this is to enable
+ us to nuke nodes that were left locked due to e.g. infrastructure failures
+ and return them to the pool.
+
+ :param owner: If non-None, return nodes locked by owner. Default is None.
+ """
+ def might_be_stale(node_dict):
+ """
+ Answer the question: "might this be a stale lock?"
+
+ The answer is yes if:
+ It is locked
+ It has a non-null description containing multiple '/' characters
+
+ ... because we really want "nodes that were locked for a particular job
+ and are still locked" and the above is currently the best way to guess.
+ """
+ desc = node_dict['description']
+ if (node_dict['locked'] is True and
+ desc is not None and desc.startswith('/') and
+ desc.count('/') > 1):
+ return True
+ return False
+
+ # Which nodes are locked for jobs?
+ nodes = list_locks(locked=True)
+ if owner is not None:
+ nodes = [node for node in nodes if node['locked_by'] == owner]
+ nodes = filter(might_be_stale, nodes)
+
+ def node_job_is_active(node, cache):
+ """
+ Is this node's job active (e.g. running or waiting)?
+
+ :param node: The node dict as returned from the lock server
+ :param cache: A set() used for caching results
+ :returns: True or False
+ """
+ description = node['description']
+ if description in cache:
+ return True
+ (name, job_id) = description.split('/')[-2:]
+ url = os.path.join(config.results_server, 'runs', name, 'jobs', job_id,
+ '')
+ resp = requests.get(url)
+ job_info = resp.json()
+ if job_info['status'] in ('running', 'waiting'):
+ cache.add(description)
+ return True
+ return False
+
+ result = list()
+ # Here we build the list of of nodes that are locked, for a job (as opposed
+ # to being locked manually for random monkeying), where the job is not
+ # running
+ active_jobs = set()
+ for node in nodes:
+ if node_job_is_active(node, active_jobs):
+ continue
+ result.append(node)
+ return result
\ No newline at end of file
--- /dev/null
+import teuthology.lock.util
+
+class TestLock(object):
+
+ def test_locked_since_seconds(self):
+ node = { "locked_since": "2013-02-07 19:33:55.000000" }
+ assert teuthology.lock.util.locked_since_seconds(node) > 3600
--- /dev/null
+import datetime
+import json
+import logging
+
+from teuthology import misc
+import teuthology.provision.downburst
+
+log = logging.getLogger(__name__)
+
+
+def vps_version_or_type_valid(machine_type, os_type, os_version):
+ """
+ Check os-type and os-version parameters when locking a vps.
+ Os-type will always be set (defaults to ubuntu).
+
+ In the case where downburst does not handle list-json (an older version
+ of downburst, for instance), a message is printed and this checking
+ is skipped (so that this code should behave as it did before this
+ check was added).
+ """
+ if not machine_type == 'vps':
+ return True
+ if os_type is None or os_version is None:
+ # we'll use the defaults provided by provision.create_if_vm
+ # later on during provisioning
+ return True
+ valid_os_and_version = \
+ teuthology.provision.downburst.get_distro_from_downburst()
+ if os_type not in valid_os_and_version:
+ log.error("os-type '%s' is invalid. Try one of: %s",
+ os_type,
+ ', '.join(valid_os_and_version.keys()))
+ return False
+ if not validate_distro_version(os_version,
+ valid_os_and_version[os_type]):
+ log.error(
+ "os-version '%s' is invalid for os-type '%s'. Try one of: %s",
+ os_version,
+ os_type,
+ ', '.join(valid_os_and_version[os_type]))
+ return False
+ return True
+
+
+def validate_distro_version(version, supported_versions):
+ """
+ Return True if the version is valid. For Ubuntu, possible
+ supported version values are of the form '12.04 (precise)' where
+ either the number of the version name is acceptable.
+ """
+ if version in supported_versions:
+ return True
+ for parts in supported_versions:
+ part = parts.split('(')
+ if len(part) == 2:
+ if version == part[0]:
+ return True
+ if version == part[1][0:len(part[1])-1]:
+ return True
+
+
+def json_matching_statuses(json_file_or_str, statuses):
+ """
+ Filter statuses by json dict in file or fragment; return list of
+ matching statuses. json_file_or_str must be a file containing
+ json or json in a string.
+ """
+ try:
+ open(json_file_or_str, 'r')
+ except IOError:
+ query = json.loads(json_file_or_str)
+ else:
+ query = json.load(json_file_or_str)
+
+ if not isinstance(query, dict):
+ raise RuntimeError('--json-query must be a dict')
+
+ return_statuses = list()
+ for status in statuses:
+ for k, v in query.iteritems():
+ if not misc.is_in_dict(k, v, status):
+ break
+ else:
+ return_statuses.append(status)
+
+ return return_statuses
+
+
+def winnow(statuses, arg, status_key, func=None):
+ """
+ Call with a list of statuses, and the ctx.<key>
+ 'arg' that you may want to filter by.
+ If arg is not None, filter statuses by either:
+
+ 1) func=None: filter by status[status_key] == arg
+ remove any status that fails
+
+ 2) func=<filter function that takes status>: remove any
+ status for which func returns False
+
+ Return the possibly-smaller set of statuses.
+ """
+
+ if arg is not None:
+ if func:
+ statuses = [_status for _status in statuses
+ if func(_status)]
+ else:
+ statuses = [_status for _status in statuses
+ if _status[status_key] == arg]
+
+ return statuses
+
+
+def locked_since_seconds(node):
+ now = datetime.datetime.now()
+ since = datetime.datetime.strptime(
+ node['locked_since'], '%Y-%m-%d %H:%M:%S.%f')
+ return (now - since).total_seconds()
+
+
Generator of ids.
Each call returns the next possible role of the type specified.
- :param roles_for host: list of roles possible
+ :param roles_for_host: list of roles possible
:param type_: type of role
"""
for role in cluster_roles_of_type(roles_for_host, type_, None):
Generator of roles.
Each call returns the next possible role of the type specified.
- :param roles_for host: list of roles possible
+ :param roles_for_host: list of roles possible
:param type_: type of role
:param cluster: cluster name
"""
type specified.
:param cluster: Cluster extracted from the ctx.
- :type_: role type
+ :param type_: role type
"""
for _, roles_for_host in cluster.remotes.iteritems():
for id_ in roles_of_type(roles_for_host, type_):
import logging
import os
import subprocess
+
import yaml
import teuthology
-
-from ..config import config, FakeNamespace
-from ..lock import (
- list_locks, locked_since_seconds, unlock_one, find_stale_locks, get_status,
- is_vm
+from teuthology.lock.ops import unlock_one
+from teuthology.lock.query import is_vm, list_locks, \
+ find_stale_locks
+from teuthology.lock.util import locked_since_seconds
+from .actions import (
+ check_console, clear_firewall, shutdown_daemons, remove_installed_packages,
+ reboot, remove_osd_mounts, remove_osd_tmpfs, kill_hadoop,
+ remove_ceph_packages, synch_clocks, unlock_firmware_repo,
+ remove_configuration_files, undo_multipath, reset_syslog_dir,
+ remove_ceph_data, remove_testing_tree, remove_yum_timedhosts,
+ kill_valgrind,
)
+from ..config import config, FakeNamespace
from ..misc import (
canonicalize_hostname, config_file, decanonicalize_hostname, merge_configs,
get_user, sh
from ..parallel import parallel
from ..task.internal import check_lock, add_remotes, connect
-from .actions import (
- check_console, clear_firewall, shutdown_daemons, remove_installed_packages,
- reboot, remove_osd_mounts, remove_osd_tmpfs, kill_hadoop,
- remove_ceph_packages, synch_clocks,
- unlock_firmware_repo, remove_configuration_files, undo_multipath,
- reset_syslog_dir, remove_ceph_data, remove_testing_tree,
- remove_yum_timedhosts, kill_valgrind,
-)
-
log = logging.getLogger(__name__)
import shutil
import teuthology.lock
+import teuthology.lock.cli
+import teuthology.lock.query
+import teuthology.lock.util
import teuthology.nuke
import teuthology.misc
import teuthology.schedule
self.wait_worker()
log = self.get_teuthology_log()
assert "teuthology.run:FAIL" in log
- locks = teuthology.lock.list_locks(locked=True)
+ locks = teuthology.lock.query.list_locks(locked=True)
assert len(locks) == 0
class TestSchedule(Integration):
def test_main(self):
args = scripts.lock.parse_args(self.options + ['--lock'])
- assert teuthology.lock.main(args) == 0
+ assert teuthology.lock.cli.main(args) == 0
def test_lock_unlock(self):
for image in teuthology.openstack.OpenStack.image2url.keys():
['--lock-many', '1',
'--os-type', os_type,
'--os-version', os_version])
- assert teuthology.lock.main(args) == 0
- locks = teuthology.lock.list_locks(locked=True)
+ assert teuthology.lock.cli.main(args) == 0
+ locks = teuthology.lock.query.list_locks(locked=True)
assert len(locks) == 1
args = scripts.lock.parse_args(self.options +
['--unlock', locks[0]['name']])
- assert teuthology.lock.main(args) == 0
+ assert teuthology.lock.cli.main(args) == 0
def test_list(self, capsys):
args = scripts.lock.parse_args(self.options + ['--list', '--all'])
- teuthology.lock.main(args)
+ teuthology.lock.cli.main(args)
out, err = capsys.readouterr()
assert 'machine_type' in out
assert 'openstack' in out
['--lock-many', '1',
'--os-type', os_type,
'--os-version', os_version])
- assert teuthology.lock.main(args) == 0
- locks = teuthology.lock.list_locks(locked=True)
+ assert teuthology.lock.cli.main(args) == 0
+ locks = teuthology.lock.query.list_locks(locked=True)
logging.info('list_locks = ' + str(locks))
assert len(locks) == 1
ctx = argparse.Namespace(name=None,
owner=locks[0]['locked_by'],
teuthology_config={})
teuthology.nuke.nuke(ctx, should_unlock=True)
- locks = teuthology.lock.list_locks(locked=True)
+ locks = teuthology.lock.query.list_locks(locked=True)
assert len(locks) == 0
import sys
import time
-from teuthology import lock
+import teuthology.lock.query
+import teuthology.lock.util
from teuthology.config import config
from ..exceptions import ConsoleError
raise RuntimeError("libvirt not found")
self.shortname = remote.getShortName(name)
- status_info = lock.get_status(self.shortname)
+ status_info = teuthology.lock.query.get_status(self.shortname)
try:
- if lock.is_vm(status=status_info):
+ if teuthology.lock.query.is_vm(status=status_info):
phys_host = status_info['vm_host']['name'].split('.')[0]
except TypeError:
return
"""
Support for paramiko remote objects.
"""
+import teuthology.lock.query
+import teuthology.lock.util
from . import run
from .opsys import OS
import connection
import console
-from teuthology import lock
-
log = logging.getLogger(__name__)
@property
def machine_type(self):
if not getattr(self, '_machine_type', None):
- remote_info = lock.get_status(self.hostname)
+ remote_info = teuthology.lock.query.get_status(self.hostname)
if not remote_info:
return None
self._machine_type = remote_info.get("machine_type", None)
"""
if self.os.package_type != 'rpm':
return
- if lock.is_vm(self.shortname):
+ if teuthology.lock.query.is_vm(self.shortname):
return
self.run(args="sudo chcon {con} {path}".format(
con=context, path=file_path))
@property
def is_vm(self):
if not hasattr(self, '_is_vm'):
- self._is_vm = lock.is_vm(self.name)
+ self._is_vm = teuthology.lock.query.is_vm(self.name)
return self._is_vm
def __del__(self):
"""
Return either VirtualConsole or PhysicalConsole depending on name.
"""
- if lock.is_vm(name):
+ if teuthology.lock.query.is_vm(name):
return console.VirtualConsole(name)
return console.PhysicalConsole(
name, ipmiuser, ipmipass, ipmidomain, logfile, timeout)
import logging
-from ..misc import decanonicalize_hostname, get_distro, get_distro_version
-from ..lock import get_status, is_vm
-from .downburst import Downburst
-from .openstack import ProvisionOpenStack
+import teuthology.lock.query
+from ..misc import decanonicalize_hostname, get_distro, get_distro_version
import cloud
+import downburst
+import openstack
log = logging.getLogger(__name__)
if _downburst:
status_info = _downburst.status
else:
- status_info = get_status(machine_name)
+ status_info = teuthology.lock.query.get_status(machine_name)
shortname = decanonicalize_hostname(machine_name)
machine_type = status_info['machine_type']
os_type = get_distro(ctx)
os_version = get_distro_version(ctx)
- if not is_vm(status=status_info):
+ if not teuthology.lock.query.is_vm(status=status_info):
return False
if machine_type in cloud.get_types():
'Usage of a custom downburst config has been deprecated.'
)
- dbrst = _downburst or Downburst(name=machine_name, os_type=os_type,
- os_version=os_version, status=status_info)
+ dbrst = _downburst or \
+ downburst.Downburst(name=machine_name, os_type=os_type,
+ os_version=os_version, status=status_info)
return dbrst.create()
if _downburst:
status_info = _downburst.status
else:
- status_info = get_status(machine_name)
- if not status_info or not is_vm(status=status_info):
+ status_info = teuthology.lock.query.get_status(machine_name)
+ if not status_info or not teuthology.lock.query.is_vm(status=status_info):
return True
if user is not None and user != status_info['locked_by']:
msg = "Tried to destroy {node} as {as_user} but it is locked " + \
machine_type = status_info.get('machine_type')
shortname = decanonicalize_hostname(machine_name)
if machine_type == 'openstack':
- return ProvisionOpenStack().destroy(shortname)
+ return openstack.ProvisionOpenStack().destroy(shortname)
elif machine_type in cloud.get_types():
return cloud.get_provisioner(
machine_type, shortname, None, None).destroy()
- dbrst = _downburst or Downburst(name=machine_name, os_type=None,
- os_version=None, status=status_info)
+ dbrst = _downburst or \
+ downburst.Downburst(name=machine_name, os_type=None,
+ os_version=None, status=status_info)
return dbrst.destroy()
+import json
import logging
import os
import subprocess
from ..config import config
from ..contextutil import safe_while
from ..misc import decanonicalize_hostname
-from ..lock import get_status
-
+from teuthology.lock import query
log = logging.getLogger(__name__)
self.name = name
self.os_type = os_type
self.os_version = os_version
- self.status = status or get_status(self.name)
+ self.status = status or query.get_status(self.name)
self.config_path = None
self.user_path = None
self.user = user
def __del__(self):
self.remove_config()
+
+
+def get_distro_from_downburst():
+ """
+ Return a table of valid distros.
+
+ If downburst is in path use it. If either downburst is unavailable,
+ or if downburst is unable to produce a json list, then use a default
+ table.
+ """
+ default_table = {u'rhel_minimal': [u'6.4', u'6.5'],
+ u'fedora': [u'17', u'18', u'19', u'20', u'22'],
+ u'centos': [u'6.3', u'6.4', u'6.5', u'7.0',
+ u'7.2'],
+ u'centos_minimal': [u'6.4', u'6.5'],
+ u'ubuntu': [u'8.04(hardy)', u'9.10(karmic)',
+ u'10.04(lucid)', u'10.10(maverick)',
+ u'11.04(natty)', u'11.10(oneiric)',
+ u'12.04(precise)', u'12.10(quantal)',
+ u'13.04(raring)', u'13.10(saucy)',
+ u'14.04(trusty)', u'utopic(utopic)',
+ u'16.04(xenial)'],
+ u'sles': [u'11-sp2'],
+ u'debian': [u'6.0', u'7.0', u'8.0']}
+ executable_cmd = downburst_executable()
+ if not executable_cmd:
+ log.warn("Downburst not found!")
+ log.info('Using default values for supported os_type/os_version')
+ return default_table
+ try:
+ output = subprocess.check_output([executable_cmd, 'list-json'])
+ downburst_data = json.loads(output)
+ return downburst_data
+ except (subprocess.CalledProcessError, OSError):
+ log.exception("Error calling downburst!")
+ log.info('Using default values for supported os_type/os_version')
+ return default_table
\ No newline at end of file
ctx = self.ctx
status = self.status
- dbrst = provision.Downburst(
+ dbrst = provision.downburst.Downburst(
name, ctx.os_type, ctx.os_version, status)
dbrst.executable = '/fake/path'
dbrst.build_config = MagicMock(name='build_config')
ctx = self.ctx
status = self.status
- dbrst = provision.Downburst(
+ dbrst = provision.downburst.Downburst(
name, ctx.os_type, ctx.os_version, status)
dbrst.destroy = MagicMock(name='destroy')
dbrst.destroy.return_value = True
status = self.status
status['locked_by'] = 'user@a'
- dbrst = provision.Downburst(
+ dbrst = provision.downburst.Downburst(
name, ctx.os_type, ctx.os_version, status)
dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError)
status = self.status
status['description'] = 'desc_a'
- dbrst = provision.Downburst(
+ dbrst = provision.downburst.Downburst(
name, ctx.os_type, ctx.os_version, status)
dbrst.destroy = MagicMock(name='destroy')
dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError)
_downburst=dbrst)
assert result is False
- @patch('teuthology.provision_executable')
+ @patch('teuthology.provision.downburst.downburst_executable')
def test_create_fails_without_executable(self, m_exec):
name = self.name
ctx = self.ctx
status = self.status
m_exec.return_value = ''
- dbrst = provision.Downburst(
+ dbrst = provision.downburst.Downburst(
name, ctx.os_type, ctx.os_version, status)
result = dbrst.create()
assert result is False
- @patch('teuthology.provision_executable')
+ @patch('teuthology.provision.downburst.downburst_executable')
def test_destroy_fails_without_executable(self, m_exec):
name = self.name
ctx = self.ctx
status = self.status
m_exec.return_value = ''
- dbrst = provision.Downburst(
+ dbrst = provision.downburst.Downburst(
name, ctx.os_type, ctx.os_version, status)
result = dbrst.destroy()
assert result is False
)
assert result == "some json"
- @patch('teuthology.suite.util.lock')
- def test_get_arch_fail(self, m_lock):
- m_lock.list_locks.return_value = False
+ @patch('teuthology.lock.query')
+ def test_get_arch_fail(self, m_query):
+ m_query.list_locks.return_value = False
util.get_arch('magna')
- m_lock.list_locks.assert_called_with(machine_type="magna", count=1)
+ m_query.list_locks.assert_called_with(machine_type="magna", count=1)
- @patch('teuthology.suite.util.lock')
- def test_get_arch_success(self, m_lock):
- m_lock.list_locks.return_value = [{"arch": "arch"}]
+ @patch('teuthology.lock.query')
+ def test_get_arch_success(self, m_query):
+ m_query.list_locks.return_value = [{"arch": "arch"}]
result = util.get_arch('magna')
- m_lock.list_locks.assert_called_with(
+ m_query.list_locks.assert_called_with(
machine_type="magna",
count=1
)
from email.mime.text import MIMEText
-from .. import lock
+import teuthology.lock.query
+import teuthology.lock.util
from .. import repo_utils
from ..config import config
:returns: A string or None
"""
- result = lock.list_locks(machine_type=machine_type, count=1)
+ result = teuthology.lock.query.list_locks(machine_type=machine_type, count=1)
if not result:
log.warn("No machines found with machine_type %s!", machine_type)
else:
import yaml
import subprocess
-from teuthology import lock
+import teuthology.lock.ops
from teuthology import misc
from teuthology.packaging import get_builder_project
from teuthology import report
def push():
for rem in ctx.cluster.remotes.keys():
info = rem.inventory_info
- lock.update_inventory(info)
+ teuthology.lock.ops.update_inventory(info)
try:
push()
except Exception:
import logging
-from teuthology import lock
+import teuthology.lock.query
+import teuthology.lock.util
from teuthology.config import config as teuth_config
return
log.info('Checking locks...')
for machine in ctx.config['targets'].iterkeys():
- status = lock.get_status(machine)
+ status = teuthology.lock.query.get_status(machine)
log.debug('machine status is %s', repr(status))
assert status is not None, \
'could not read lock status for {name}'.format(name=machine)
import time
import yaml
-from teuthology import lock
+import teuthology.lock.keys
+import teuthology.lock.ops
+import teuthology.lock.query
+import teuthology.lock.util
from teuthology import misc
from teuthology import provision
from teuthology import report
requested = total_requested
while True:
# get a candidate list of machines
- machines = lock.list_locks(machine_type=machine_type, up=True,
- locked=False, count=requested + reserved)
+ machines = teuthology.lock.query.list_locks(machine_type=machine_type, up=True,
+ locked=False, count=requested + reserved)
if machines is None:
if ctx.block:
log.error('Error listing machines, trying again')
(reserved, requested, len(machines)))
try:
- newly_locked = lock.lock_many(ctx, requested, machine_type,
- ctx.owner, ctx.archive, os_type,
- os_version, arch)
+ newly_locked = teuthology.lock.ops.lock_many(ctx, requested, machine_type,
+ ctx.owner, ctx.archive, os_type,
+ os_version, arch)
except Exception:
# Lock failures should map to the 'dead' status instead of 'fail'
set_status(ctx.summary, 'dead')
if len(all_locked) == total_requested:
vmlist = []
for lmach in all_locked:
- if lock.is_vm(lmach):
+ if teuthology.lock.query.is_vm(lmach):
vmlist.append(lmach)
if vmlist:
log.info('Waiting for virtual machines to come up')
full_name = misc.canonicalize_hostname(guest)
provision.destroy_if_vm(ctx, full_name)
provision.create_if_vm(ctx, full_name)
- if lock.do_update_keys(keys_dict):
+ if teuthology.lock.keys.do_update_keys(keys_dict):
log.info("Error in virtual machine keys")
newscandict = {}
for dkey in all_locked.iterkeys():
- stats = lock.get_status(dkey)
+ stats = teuthology.lock.query.get_status(dkey)
newscandict[dkey] = stats['ssh_pub_key']
ctx.config['targets'] = newscandict
else:
if get_status(ctx.summary) == 'pass' or unlock_on_failure:
log.info('Unlocking machines...')
for machine in ctx.config['targets'].iterkeys():
- lock.unlock_one(ctx, machine, ctx.owner, ctx.archive)
+ teuthology.lock.ops.unlock_one(ctx, machine, ctx.owner, ctx.archive)
+++ /dev/null
-from teuthology import lock
-
-
-class TestLock(object):
-
- def test_locked_since_seconds(self):
- node = { "locked_since": "2013-02-07 19:33:55.000000" }
- assert lock.locked_since_seconds(node) > 3600
from mock import patch, Mock
-from .. import lock
+import teuthology.lock.util
+from .. import provision
class TestVpsOsVersionParamCheck(object):
self.fake_ctx.os_type = 'ubuntu'
self.fake_ctx.os_version = 'precise'
with patch.multiple(
- lock.provision.downburst,
+ provision.downburst,
downburst_executable=self.fake_downburst_executable,
):
- check_value = lock.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
assert check_value
self.fake_ctx.os_type = 'ubuntu'
self.fake_ctx.os_version = '12.04'
with patch.multiple(
- lock.provision.downburst,
+ provision.downburst,
downburst_executable=self.fake_downburst_executable,
):
- check_value = lock.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
assert check_value
def test_mixup(self):
self.fake_ctx.os_type = '6.5'
self.fake_ctx.os_version = 'rhel'
with patch.multiple(
- lock.provision.downburst,
+ provision.downburst,
downburst_executable=self.fake_downburst_executable,
):
- check_value = lock.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
assert not check_value
def test_bad_type(self):
self.fake_ctx.os_type = 'aardvark'
self.fake_ctx.os_version = '6.5'
with patch.multiple(
- lock.provision.downburst,
+ provision.downburst,
downburst_executable=self.fake_downburst_executable,
):
- check_value = lock.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
assert not check_value
def test_bad_version(self):
self.fake_ctx.os_type = 'rhel'
self.fake_ctx.os_version = 'vampire_bat'
with patch.multiple(
- lock.provision.downburst,
+ provision.downburst,
downburst_executable=self.fake_downburst_executable,
):
- check_value = lock.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
assert not check_value
-
projects / teuthology.git / commitdiff