From 8129bffb17141abafadf826fa1bc429d8f3c9dd2 Mon Sep 17 00:00:00 2001 From: Warren Usui Date: Tue, 2 Apr 2013 18:27:38 -0700 Subject: [PATCH] Implement full reinstallation of a VM system. Downburst create is used to reinstall a VM when it is locked. Downburst destroy is used to remove a VM when it is unlocked. Host keys are regenerated on each vm instantiation, so the keys need to be checked prior to use. If needed, qa-ceph-chef is run on newly installed systems to insure that they are fully functional. Signed-off-by: Warren Usui --- README.rst | 63 +++++++++++++++++++++++++ teuthology/lock.py | 82 +++++++++++++++++++++++++++++++-- teuthology/run.py | 1 + teuthology/task/edit_sudoers.sh | 10 ++++ teuthology/task/internal.py | 34 ++++++++++++++ 5 files changed, 186 insertions(+), 4 deletions(-) create mode 100755 teuthology/task/edit_sudoers.sh diff --git a/README.rst b/README.rst index 92e767febc920..b1e518be76110 100644 --- a/README.rst +++ b/README.rst @@ -209,3 +209,66 @@ run directories. If not specified, this defaults to: ``/tmp/cephtest``. The ``test_path`` option will set the complete path to use for the test directory. This allows for the old behavior, where ``/tmp/cephtest`` was used as the sandbox directory. + + +VIRTUAL MACHINE SUPPORT +======= ======= ======= + +Teuthology also supports virtual machines, which can function like +physical machines but differ in the following ways: + +VPSHOST: + +A new entry, vpshost, has been added to the teuthology database of +available machines. For physical machines, this value is null. For +virtual machines, this entry is the name of the physical machine that +that virtual machine resides on. + +There are fixed "slots" for virtual machines that appear in the teuthology +database. These slots have a machine type of vps and can be locked like +any other machine. The existence of a vpshost field is how teuthology +knows whether or not a database entry represents a physical or a virtual +machine. + +DOWNBURST: + +When a virtual machine is locked, downburst is run on that machine to +install a new image. This allows the user to set different virtual +OSes to be installed on the newly created virtual machine. Currently +the default virtual machine is ubuntu (precise). A different vm installation +can be set using the --vm-type option in teuthology.lock. + +When a virtual machine is unlocked, downburst destroys the image on the +machine. + +Temporary yaml files are used to downburst a virtual machine. A typical +yaml file will look like this: + +downburst: + cpus: 1 + disk-size: 30G + distro: centos + networks: + - {source: front} + ram: 4G + +These values are used by downburst to create the virtual machine. + +HOST KEYS: + +Because teuthology reinstalls a new machine, a new hostkey is generated. +After locking, once a connection is established to the new machine, +teuthology-lock with the --list or --list-targets options will display +the new keys. When vps machines are locked using the --lock-many option, +a message is displayed indicating that --list-targets should be run later. + +CEPH-QA-CHEF: + +Once teuthology starts after a new vm is installed, teuthology +checks for the existence of /ceph-qa-ready. If this file is not +present, ceph-qa-chef is run when teuthology first comes up. + +ASSUMPTIONS: + +It is assumed that downburst is on the user's PATH. + diff --git a/teuthology/lock.py b/teuthology/lock.py index b63260ceae534..eb698c2aba80f 100644 --- a/teuthology/lock.py +++ b/teuthology/lock.py @@ -7,6 +7,8 @@ import urllib import yaml import re import collections +import os +import tempfile from teuthology import misc as teuthology @@ -222,6 +224,11 @@ Lock, unlock, or query lock status of machines. nargs='*', help='machines to operate on', ) + parser.add_argument( + '--vm-type', + default='ubuntu', + help='virtual machine type', + ) ctx = parser.parse_args() @@ -279,7 +286,18 @@ Lock, unlock, or query lock status of machines. statuses = [get_status(ctx, machine) for machine in machines] else: statuses = list_locks(ctx) - + vmachines = [] + + for vmachine in statuses: + if vmachine['vpshost']: + if vmachine['locked']: + vmachines.append(vmachine['name']) + if vmachines: + scan_for_locks(ctx, vmachines) + if machines: + statuses = [get_status(ctx, machine) for machine in machines] + else: + statuses = list_locks(ctx) if statuses: if not machines and ctx.owner is None and not ctx.all: ctx.owner = teuthology.get_user() @@ -331,6 +349,9 @@ Lock, unlock, or query lock status of machines. return ret else: machines_to_update.append(machine) + status_info = get_status(ctx,machine) + if status_info['vpshost']: + do_create(ctx, machine, status_info['vpshost']) elif ctx.unlock: for machine in machines: if not unlock(ctx, machine, user): @@ -339,13 +360,21 @@ Lock, unlock, or query lock status of machines. return ret else: machines_to_update.append(machine) + status_info = get_status(ctx,machine) + if status_info['vpshost']: + do_destroy(machine, status_info['vpshost']) elif ctx.num_to_lock: result = lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user) if not result: ret = 1 else: machines_to_update = result.keys() - print yaml.safe_dump(dict(targets=result), default_flow_style=False) + if ctx.machine_type == 'vps': + print "Locks successful" + print "Unable to display keys at this time (virtual machines are rebooting)." + print "Please run teuthology-lock --list-targets once these machines come up." + else: + print yaml.safe_dump(dict(targets=result), default_flow_style=False) elif ctx.update: assert ctx.desc is not None or ctx.status is not None, \ 'you must specify description or status to update' @@ -404,7 +433,6 @@ to run on, or use -a to check all of them automatically. if ctx.all: assert not ctx.targets and not ctx.machines, \ 'You can\'t specify machines with the --all option' - machines = [canonicalize_hostname(m) for m in ctx.machines] if ctx.targets: @@ -417,7 +445,10 @@ to run on, or use -a to check all of them automatically. machines.append(t) except IOError, e: raise argparse.ArgumentTypeError(str(e)) + + return scan_for_locks(ctx, machines) +def scan_for_locks(ctx, machines): locks = list_locks(ctx) current_locks = {} for lock in locks: @@ -429,7 +460,6 @@ to run on, or use -a to check all of them automatically. for i, machine in enumerate(machines): if '@' in machine: _, machines[i] = machine.rsplit('@') - args = ['ssh-keyscan'] args.extend(machines) p = subprocess.Popen( @@ -478,3 +508,47 @@ def do_summary(ctx): print " --- ---" print "{cnt:12d} {up:3d}".format(cnt = total_count, up = total_up) +def decanonicalize_hostname(s): + if re.match('ubuntu@.*\.front\.sepia\.ceph\.com', s): + s = s[len('ubuntu@'): -len('.front.sepia.ceph.com')] + return s +# +# Use downburst to create a virtual machine +# +def do_create(ctx, machine_name, phys_host): + vmType = ctx.vm_type + createMe = decanonicalize_hostname(machine_name) + with tempfile.NamedTemporaryFile() as tmp: + fileInfo1 = {} + fileInfo1['disk-size'] = '30G' + fileInfo1['ram'] = '4G' + fileInfo1['cpus'] = 1; + fileInfo1['networks'] = [{'source' : 'front'}] + fileInfo1['distro'] = vmType.lower() + fileOwt = {'downburst': fileInfo1} + yaml.safe_dump(fileOwt,tmp) + metadata = "--meta-data=%s" % tmp.name + p = subprocess.Popen(['downburst', '-c', phys_host, + 'create', metadata, createMe], + stdout=subprocess.PIPE,stderr=subprocess.PIPE,) + owt,err = p.communicate() + if err: + log.info("Downburst command to create %s may have failed: %s", + (machine_name,err)) + else: + log.info("%s created: %s" % (machine_name,owt)) + return True +# +# Use downburst to destroy a virtual machine +# +def do_destroy(machine_name, phys_host): + destroyMe = decanonicalize_hostname(machine_name) + p = subprocess.Popen(['downburst', '-c', phys_host, + 'destroy', destroyMe], + stdout=subprocess.PIPE,stderr=subprocess.PIPE,) + owt,err = p.communicate() + if err: + log.info("Error occurred while deleting %s" % destroyMe) + else: + log.info("%s destroyed: %s" % (machine_name,owt)) + diff --git a/teuthology/run.py b/teuthology/run.py index 6bede22629d05..f5dc5d1d73415 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -163,6 +163,7 @@ def main(): {'internal.connect': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, + {'internal.vm_setup': None}, ]) if 'kernel' in ctx.config: init_tasks.append({'kernel': ctx.config['kernel']}) diff --git a/teuthology/task/edit_sudoers.sh b/teuthology/task/edit_sudoers.sh new file mode 100755 index 0000000000000..6ab40a5d80d79 --- /dev/null +++ b/teuthology/task/edit_sudoers.sh @@ -0,0 +1,10 @@ +#! /bin/sh + +sudo vi -e /etc/sudoers <