Implement single-job killing

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/scripts/kill.py b/scripts/kill.py
index 9333b0672dd3469522e8b34ff4afacd819def1ef..032b4fdc0e913989f35766f6c02c7ddddb6ca782 100644 (file)
--- a/scripts/kill.py
+++ b/scripts/kill.py
@@ -5,6 +5,7 @@ import teuthology.kill
 
 doc = """
 usage: teuthology-kill [-h] [-a ARCHIVE] -r RUN
+       teuthology-kill [-h] [-a ARCHIVE] -r RUN -j JOB
        teuthology-kill [-h] -o OWNER -m MACHINE_TYPE -r RUN
 
 Kill running teuthology jobs:
@@ -17,7 +18,8 @@ optional arguments:
   -a ARCHIVE, --archive ARCHIVE
                         The base archive directory
                         [default: {archive_base}]
-  -r, --run RUN     The name(s) of the run(s) to kill
+  -r, --run RUN         The name(s) of the run(s) to kill
+  -j, --job JOB         The job_id of the job to kill
   -o, --owner OWNER     The owner of the job(s)
   -m, --machine_type MACHINE_TYPE
                         The type of machine the job(s) are running on

diff --git a/teuthology/kill.py b/teuthology/kill.py
index 3024ac9aa369f4efd0faa7bb245d7e81937df997..430fcf1f71f095452a71ba6a3a58944e105d8f65 100755 (executable)
--- a/teuthology/kill.py
+++ b/teuthology/kill.py
@@ -15,11 +15,15 @@ log = logging.getLogger(__name__)
 
 def main(args):
     run_name = args['--run']
+    job_id = args['--job']
     archive_base = args['--archive']
     owner = args['--owner']
     machine_type = args['--machine_type']
 
-    kill_run(run_name, archive_base, owner, machine_type)
+    if job_id:
+        kill_job(run_name, job_id, archive_base, owner, machine_type)
+    else:
+        kill_run(run_name, archive_base, owner, machine_type)
 
 
 def kill_run(run_name, archive_base=None, owner=None, machine_type=None):
@@ -32,7 +36,18 @@ def kill_run(run_name, archive_base=None, owner=None, machine_type=None):
 
     remove_beanstalk_jobs(run_name, machine_type)
     kill_processes(run_name, run_info.get('pids'))
-    nuke_machines(run_name, owner)
+    targets = find_targets(run_name, owner)
+    nuke_targets(targets, owner)
+
+
+def kill_job(run_name, job_id, archive_base=None, owner=None,
+             machine_type=None):
+    job_archive_dir = os.path.join(archive_base, run_name, job_id)
+    job_info = find_job_info(job_archive_dir)
+    owner = job_info['owner']
+    kill_processes(run_name, [job_info.get('pid')])
+    targets = dict(targets=job_info['targets'])
+    nuke_targets(targets, owner)
 
 
 def find_run_info(run_archive_dir):
@@ -118,15 +133,21 @@ def kill_processes(run_name, pids=None):
             subprocess.call(['sudo', 'kill', str(pid)])
 
 
+def process_matches_run(pid, run_name):
+    try:
+        p = psutil.Process(pid)
+        if run_name in p.cmdline and sys.argv[0] not in p.cmdline:
+            return True
+    except psutil.NoSuchProcess:
+        pass
+    return False
+
+
 def find_pids(run_name):
     run_pids = []
     for pid in psutil.get_pid_list():
-        try:
-            p = psutil.Process(pid)
-        except psutil.NoSuchProcess:
-            continue
-        if run_name in p.cmdline and sys.argv[0] not in p.cmdline:
-                run_pids.append(pid)
+        if process_matches_run(pid, run_name):
+            run_pids.append(pid)
     return run_pids
 
 
@@ -150,8 +171,7 @@ def find_targets(run_name, owner):
     return out_obj
 
 
-def nuke_machines(run_name, owner):
-    targets_dict = find_targets(run_name, owner)
+def nuke_targets(targets_dict, owner):
     targets = targets_dict.get('targets')
     if not targets:
         log.info("No locked machines. Not nuking anything")