From 4e2ec6fd8be6816d677a008694ec75c970849257 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Tue, 9 Aug 2011 15:42:17 -0700 Subject: [PATCH] Add interactive-on-error, to pause and explore on error. Closes: http://tracker.newdream.net/issues/1291 --- README.rst | 22 ++++++++++++++++++++++ teuthology/run_tasks.py | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/README.rst b/README.rst index 0330aa65c28ed..2fdb134a09018 100644 --- a/README.rst +++ b/README.rst @@ -165,3 +165,25 @@ reverse order, both on success and failure. A nice way of writing context managers is the ``contextlib.contextmanager`` decorator; look for that string in the existing tasks to see examples, and note where they use ``yield``. + + +Troubleshooting +=============== + +Sometimes when a bug triggers, instead of automatic cleanup, you want +to explore the system as is. Adding a top-level:: + + interactive-on-error: true + +as a config file for ``teuthology`` will make that possible. With that +option, any *task* that fails, will have the ``interactive`` task +called after it. This means that before any cleanup happens, you get a +change to inspect the system -- both through Teuthology and via extra +SSH connections -- and the cleanup completes only when you choose so. +Just exit the interactive Python session to continue the cleanup. + +TODO: this only catches exceptions *between* the tasks. If a task +calls multiple subtasks, e.g. with ``contextutil.nested``, those +cleanups *will* be performed. Later on, we can let tasks communicate +the subtasks they wish to invoke to the top-level runner, avoiding +this issue. diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py index fde6cb6f8a7e8..dcd94689e6f69 100644 --- a/teuthology/run_tasks.py +++ b/teuthology/run_tasks.py @@ -29,6 +29,10 @@ def run_tasks(tasks, ctx): except: ctx.summary['success'] = False log.exception('Saw exception from tasks') + if ctx.config.get('interactive-on-error'): + from .task import interactive + log.warning('Saw failure, going into interactive mode...') + interactive.task(ctx=ctx, config=None) finally: try: exc_info = sys.exc_info() -- 2.39.5