Add teuthology-kill

Currently implements killing suites-runs.

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/requirements.txt b/requirements.txt
index 7da279c08e2ce93a0df921400579bdec18e127b4..e5c92cb0e6d93e46781e022b0b5e1dd8fbf67793 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,6 +14,7 @@ pexpect
 requests == 0.14.0
 raven
 web.py
+docopt
 
 # Test Dependencies
 # nose >=1.0.0

diff --git a/scripts/kill.py b/scripts/kill.py
new file mode 100644 (file)
index 0000000..d150993
--- /dev/null
+++ b/scripts/kill.py
@@ -0,0 +1,27 @@
+import docopt
+
+import teuthology.kill
+
+doc = """
+usage: teuthology-kill [-h] -a ARCHIVE -s SUITE
+       teuthology-kill [-h] -o OWNER -m MACHINE_TYPE -s SUITE
+
+Kill running teuthology jobs:
+1. Removes any queued jobs from the beanstalk queue
+2. Kills any running jobs
+3. Nukes any machines involved
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -a ARCHIVE, --archive ARCHIVE
+                        The base archive directory
+  -s, --suite SUITE     The name(s) of the suite(s) to kill
+  -o, --owner OWNER     The owner of the job(s)
+  -m, --machine_type MACHINE_TYPE
+                        The type of machine the job(s) are running on
+"""
+
+
+def main():
+    args = docopt.docopt(doc)
+    teuthology.kill.main(args)

diff --git a/setup.py b/setup.py
index a1cb5753a90c8e2d1502e05905a4a4ce418b3d72..df95fdb1fe2688526bfe0063ffe0d78ebad1dbdf 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -27,6 +27,7 @@ setup(
             'teuthology-coverage = scripts.coverage:main',
             'teuthology-results = scripts.results:main',
             'teuthology-report = scripts.report:main',
+            'teuthology-kill = scripts.kill:main',
             ],
         },
 

diff --git a/teuthology/kill.py b/teuthology/kill.py
new file mode 100755 (executable)
index 0000000..a7b585c
--- /dev/null
+++ b/teuthology/kill.py
@@ -0,0 +1,148 @@
+#!/usr/bin/python
+import os
+import sys
+import beanstalkc
+import yaml
+import psutil
+import subprocess
+import tempfile
+
+from .config import config
+
+
+def main(args):
+    suite_name = args['--suite']
+    archive_base = args['--archive']
+    owner = args['--owner']
+    machine_type = args['--machine_type']
+
+    kill_suite(suite_name, archive_base, owner, machine_type)
+    #if args.suite:
+    #    for suite_name in args.suite:
+    #        kill_suite(args.archive, suite_name)
+
+
+def kill_suite(suite_name, archive_base=None, owner=None, machine_type=None):
+    if archive_base:
+        suite_archive_dir = os.path.join(archive_base, suite_name)
+        job_info = find_suite_info(suite_archive_dir)
+        machine_type = job_info['machine_type']
+        owner = job_info['owner']
+
+    remove_beanstalk_jobs(suite_name, machine_type)
+    kill_processes(suite_name)
+    nuke_machines(suite_name, owner)
+
+
+def find_suite_info(suite_archive_dir):
+    job_info = {}
+    for job_dir in os.listdir(suite_archive_dir):
+        if os.path.isdir(job_dir):
+            job_info = find_job_info(job_dir)
+            if job_info:
+                break
+    return job_info
+
+
+def find_job_info(job_archive_dir):
+    info_file = os.path.join(job_archive_dir, 'info.yaml')
+    if os.path.isfile(info_file):
+        job_info = yaml.safe_load(open(info_file, 'r'))
+        return job_info
+    return {}
+
+
+def remove_beanstalk_jobs(suite_name, tube_name):
+    qhost = config.queue_host
+    qport = config.queue_port
+    if qhost is None or qport is None:
+        raise RuntimeError(
+            'Beanstalk queue information not found in {conf_path}'.format(
+                conf_path=config.teuthology_yaml))
+    print "Checking Beanstalk Queue..."
+    beanstalk = beanstalkc.Connection(host=qhost, port=qport)
+    beanstalk.watch(tube_name)
+    beanstalk.ignore('default')
+
+    curjobs = beanstalk.stats_tube(tube_name)['current-jobs-ready']
+    if curjobs != 0:
+        x = 1
+        while x != curjobs:
+            x += 1
+            job = beanstalk.reserve(timeout=20)
+            job_config = yaml.safe_load(job.body)
+            if suite_name == job_config['name']:
+                job_id = job.stats()['id']
+                msg = "Deleting job from queue. ID: " + \
+                    "{id} Name: {name} Desc: {desc}".format(
+                        id=str(job_id),
+                        name=job_config['name'],
+                        desc=job_config['description'],
+                    )
+                print msg
+                job.delete()
+    else:
+        print "No jobs in Beanstalk Queue"
+    beanstalk.close()
+
+
+def kill_processes(suite_name):
+    suite_pids = []
+    for pid in psutil.get_pid_list():
+        try:
+            p = psutil.Process(pid)
+        except psutil.NoSuchProcess:
+            continue
+        if suite_name in p.cmdline and sys.argv[0] not in p.cmdline:
+                suite_pids.append(str(pid))
+
+    if len(suite_pids) == 0:
+        print "No teuthology processes running"
+    else:
+        print 'Killing Pids: ' + str(suite_pids)
+        for pid in suite_pids:
+            subprocess.call(['sudo', 'kill', pid])
+
+
+def nuke_machines(suite_name, owner):
+    #lock_args = argparse.Namespace(
+    #    list_targets=True,
+    #    desc_pattern='/%s/' % suite_name,
+    #    status='up',
+    #    owner=owner,
+    #)
+    proc = subprocess.Popen(
+        ['teuthology-lock',
+            '--list-targets',
+            '--desc-pattern',
+            '/' + suite_name + '/',
+            '--status',
+            'up',
+            '--owner',
+            owner],
+        stdout=subprocess.PIPE)
+    tout, terr = proc.communicate()
+
+    tmp = tempfile.NamedTemporaryFile(delete=False)
+    tmp.write(tout)
+    tmp.close()
+
+    targets = yaml.safe_load(tout)['targets']
+    nuking = []
+    for target in targets:
+        nuking.append(target.split('@')[1].split('.')[0])
+    if 'ubuntu' not in tout:
+        print 'No locked machines. Not nuking anything'
+    else:
+        print 'Nuking machines: ' + str(nuking)
+        nukeargs = [
+            '/var/lib/teuthworker/teuthology-master/virtualenv/bin/teuthology-nuke',  # noqa
+            '-t', tmp.name, '--unlock', '-r', '--owner', owner]
+        nuke = subprocess.Popen(
+            nukeargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        for line in iter(nuke.stdout.readline, ''):
+            line = line.replace('\r', '').replace('\n', '')
+            print line
+            sys.stdout.flush()
+
+    os.unlink(tmp.name)