suite: Implement --sleep-before-teardown option

With this option, all jobs in the run sleep for the given number of seconds as
the very last thing they do, regardless of success or failure.

Use case: interactive debugging

Signed-off-by: Nathan Cutler <ncutler@suse.com>

diff --git a/scripts/openstack.py b/scripts/openstack.py
index b467de4f508f16425971935cda11adf7d9bb2644..ce9f05ee9339a6c39a3a724ee19c1a6d4b7ffa17 100644 (file)
--- a/scripts/openstack.py
+++ b/scripts/openstack.py
@@ -191,5 +191,10 @@ and analyze results.
         '--ceph-qa-suite-git-url',
         help=("git clone url for ceph-qa-suite"),
     )
+    parser.add_argument(
+        '--sleep-before-teardown',
+        help='Number of seconds to sleep before tearing down the target VMs',
+        default=0
+    )
 
     return parser.parse_args(argv)

diff --git a/scripts/suite.py b/scripts/suite.py
index 44b4fac74d697d19910b9979384dc3204250eae5..2de6ee12fad8ccc9f4bf527bc12608e84de2f65f 100644 (file)
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -71,6 +71,11 @@ Standard arguments:
                               <suite_branch> to be ignored for scheduling
                               purposes, but it will still be used for test
                               running.
+  --sleep-before-teardown <seconds>
+                              Number of seconds to sleep before tearing down
+                              the test cluster(s). Use with care, as this
+                              applies to all jobs in the run.
+                              [default: 0]
 
 Scheduler arguments:
   --owner <owner>             Job owner

diff --git a/teuthology/config.py b/teuthology/config.py
index 16b698746a9ae840d5d1fc1b47fe9bf76dfb9428..965dc6f5fbe8115fe28cfc299914ce62000e5468 100644 (file)
--- a/teuthology/config.py
+++ b/teuthology/config.py
@@ -177,6 +177,7 @@ class TeuthologyConfig(YamlConfig):
                 'size': 1,
             },
         },
+        'sleep_before_teardown': 0,
     }
 
     def __init__(self, yaml_path=None):

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index 6dba83300c7e73571c8364875e7528df461f5971..3b4e4798de947c72dcc232a155a7789d5a993858 100644 (file)
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -59,14 +59,25 @@ def _import(from_package, module_name, task_name):
     return module
 
 
-def run_one_task(taskname, **kwargs):
+def run_one_task(taskname, stack, timer, **kwargs):
+    if 'log_message' in kwargs:
+        log.info(kwargs['log_message'])
+        del(kwargs['log_message'])
+    else:
+        log.info("Running task {}...".format(taskname))
+    timer.mark('%s enter' % taskname)
     taskname = taskname.replace('-', '_')
     task = get_task(taskname)
-    return task(**kwargs)
+    manager = task(**kwargs)
+    if hasattr(manager, '__enter__'):
+        stack.append((taskname, manager))
+        manager.__enter__()
 
 
 def run_tasks(tasks, ctx):
     archive_path = ctx.config.get('archive_path')
+    sleep_before_teardown = ctx.config.get('sleep_before_teardown')
+    sleep_task = { "sleep": { "duration": sleep_before_teardown } }
     if archive_path:
         timer = Timer(
             path=os.path.join(archive_path, 'timing.yaml'),
@@ -81,12 +92,11 @@ def run_tasks(tasks, ctx):
                 ((taskname, config),) = taskdict.iteritems()
             except (ValueError, AttributeError):
                 raise RuntimeError('Invalid task definition: %s' % taskdict)
-            log.info('Running task %s...', taskname)
-            timer.mark('%s enter' % taskname)
-            manager = run_one_task(taskname, ctx=ctx, config=config)
-            if hasattr(manager, '__enter__'):
-                stack.append((taskname, manager))
-                manager.__enter__()
+            run_one_task(taskname, stack, timer, ctx=ctx, config=config)
+        if sleep_before_teardown:
+            ((taskname, config),) = sleep_task.iteritems()
+            run_one_task(taskname, stack, timer, ctx=ctx, config=config,
+                log_message='Running sleep task because --sleep-before-teardown was given...')
     except BaseException as e:
         if isinstance(e, ConnectionLostError):
             # Prevent connection issues being flagged as failures
@@ -139,6 +149,10 @@ def run_tasks(tasks, ctx):
             from .task import interactive
             log.warning('Saw failure during task execution, going into interactive mode...')
             interactive.task(ctx=ctx, config=None)
+        if sleep_before_teardown:
+            ((taskname, config),) = sleep_task.iteritems()
+            run_one_task(taskname, stack, timer, ctx=ctx, config=config,
+                log_message='Running sleep task because --sleep-before-teardown was given...')
         # Throughout teuthology, (x,) = y has been used to assign values
         # from yaml files where only one entry of type y is correct.  This
         # causes failures with 'too many values to unpack.'  We want to

diff --git a/teuthology/suite/placeholder.py b/teuthology/suite/placeholder.py
index 9af2ffb78e865b55fba1a5a8204bdcd21cb47173..f029405e17b2a7a685927f34e1f345eda408f59e 100644 (file)
--- a/teuthology/suite/placeholder.py
+++ b/teuthology/suite/placeholder.py
@@ -95,6 +95,7 @@ dict_templ = {
         }
     },
     'repo': Placeholder('ceph_repo'),
+    'sleep_before_teardown': 0,
     'suite': Placeholder('suite'),
     'suite_repo': Placeholder('suite_repo'),
     'suite_relpath': Placeholder('suite_relpath'),

diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index 5c01e39ceba6621aff2209685965c85296d0441d..64d20b46e3c992bf5998b1b4f41d9850d77f888d 100644 (file)
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -273,6 +273,8 @@ class Run(object):
             job_config.email = self.args.email
         if self.args.owner:
             job_config.owner = self.args.owner
+        if self.args.sleep_before_teardown:
+            job_config.sleep_before_teardown = int(self.args.sleep_before_teardown)
         return job_config
 
     def build_base_args(self):