Add nuke-on-error option.

This lets automated jobs nuke and unlock machines after failed
tests. Each machine is nuke individually, so one down machine won't
keep others from being nuked and unlocked.

diff --git a/teuthology/nuke.py b/teuthology/nuke.py
index d4d8aec993846dc3c3a0af620166b76b7c86bb0d..73358f6314dfaf441266306ff1d8df5396397d02 100644 (file)
--- a/teuthology/nuke.py
+++ b/teuthology/nuke.py
@@ -233,6 +233,9 @@ def main():
         from teuthology.misc import get_user
         ctx.owner = get_user()
 
+    nuke(ctx, log)
+
+def nuke(ctx, log):
     from teuthology.task.internal import check_lock, connect
     check_lock(ctx, None)
     connect(ctx, None)

diff --git a/teuthology/run.py b/teuthology/run.py
index 62a631298b6376e82f0e48e0ef6554815c591145..3455daeb23d7c400ed9d613473ae48d2217be08c 100644 (file)
--- a/teuthology/run.py
+++ b/teuthology/run.py
@@ -169,6 +169,38 @@ def main():
             with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f:
                 yaml.safe_dump(ctx.summary, f, default_flow_style=False)
 
+        if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
+            from teuthology.parallel import parallel
+            with parallel() as p:
+                for target, hostkey in ctx.config['targets'].iteritems():
+                    p.spawn(
+                        nuke_and_unlock,
+                        targets={target: hostkey},
+                        owner=ctx.owner,
+                        log=log,
+                        teuth_config=ctx.teuthology_config,
+                        )
+
+def nuke_and_unlock(targets, owner, log, teuth_config,
+                    synch_clocks=True, reboot_all=True):
+    from teuthology.nuke import nuke
+    from teuthology.lock import unlock
+    ctx = argparse.Namespace(
+        config=dict(targets=targets),
+        owner=owner,
+        synch_clocks=synch_clocks,
+        reboot_all=reboot_all,
+        teuthology_config=teuth_config,
+        )
+    try:
+        nuke(ctx, log)
+    except:
+        log.exception('Could not nuke all targets in %s', targets)
+        # not re-raising the so that parallel calls aren't killed
+    else:
+        for target in targets.keys():
+            unlock(ctx, target, owner)
+
 def schedule():
     parser = argparse.ArgumentParser(description='Schedule ceph integration tests')
     parser.add_argument(