From 5d32bcae50e4d89008119d23b231db56bac3b1b0 Mon Sep 17 00:00:00 2001
From: Josh Durgin <josh.durgin@dreamhost.com>
Date: Tue, 8 Nov 2011 16:01:39 -0800
Subject: [PATCH] Add nuke-on-error option.

This lets automated jobs nuke and unlock machines after failed
tests. Each machine is nuke individually, so one down machine won't
keep others from being nuked and unlocked.
---
 teuthology/nuke.py |  3 +++
 teuthology/run.py  | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/teuthology/nuke.py b/teuthology/nuke.py
index d4d8aec993846..73358f6314dfa 100644
--- a/teuthology/nuke.py
+++ b/teuthology/nuke.py
@@ -233,6 +233,9 @@ def main():
         from teuthology.misc import get_user
         ctx.owner = get_user()
 
+    nuke(ctx, log)
+
+def nuke(ctx, log):
     from teuthology.task.internal import check_lock, connect
     check_lock(ctx, None)
     connect(ctx, None)
diff --git a/teuthology/run.py b/teuthology/run.py
index 62a631298b637..3455daeb23d7c 100644
--- a/teuthology/run.py
+++ b/teuthology/run.py
@@ -169,6 +169,38 @@ def main():
             with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f:
                 yaml.safe_dump(ctx.summary, f, default_flow_style=False)
 
+        if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
+            from teuthology.parallel import parallel
+            with parallel() as p:
+                for target, hostkey in ctx.config['targets'].iteritems():
+                    p.spawn(
+                        nuke_and_unlock,
+                        targets={target: hostkey},
+                        owner=ctx.owner,
+                        log=log,
+                        teuth_config=ctx.teuthology_config,
+                        )
+
+def nuke_and_unlock(targets, owner, log, teuth_config,
+                    synch_clocks=True, reboot_all=True):
+    from teuthology.nuke import nuke
+    from teuthology.lock import unlock
+    ctx = argparse.Namespace(
+        config=dict(targets=targets),
+        owner=owner,
+        synch_clocks=synch_clocks,
+        reboot_all=reboot_all,
+        teuthology_config=teuth_config,
+        )
+    try:
+        nuke(ctx, log)
+    except:
+        log.exception('Could not nuke all targets in %s', targets)
+        # not re-raising the so that parallel calls aren't killed
+    else:
+        for target in targets.keys():
+            unlock(ctx, target, owner)
+
 def schedule():
     parser = argparse.ArgumentParser(description='Schedule ceph integration tests')
     parser.add_argument(
-- 
2.39.5