On SSH connection loss, mark jobs as 'dead'

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index eaf59f9261470720ef0a38b9e0954c1b9a839900..fd379fe5592dd656ab63ee89e017e9d5cd444f5e 100644 (file)
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -3,6 +3,7 @@ import logging
 from .sentry import get_client as get_sentry_client
 from .misc import get_http_log_path
 from .config import config as teuth_config
+from .exceptions import ConnectionLostError
 from copy import deepcopy
 
 log = logging.getLogger(__name__)
@@ -54,6 +55,9 @@ def run_tasks(tasks, ctx):
                 stack.append((taskname, manager))
     except BaseException as e:
         ctx.summary['success'] = False
+        if isinstance(e, ConnectionLostError):
+            # Prevent connection issues being flagged as failures
+            ctx.summary['status'] = 'dead'
         if 'failure_reason' not in ctx.summary:
             ctx.summary['failure_reason'] = str(e)
         log.exception('Saw exception from tasks.')
@@ -113,6 +117,9 @@ def run_tasks(tasks, ctx):
                     suppress = manager.__exit__(*exc_info)
                 except Exception as e:
                     ctx.summary['success'] = False
+                    if isinstance(e, ConnectionLostError):
+                        # Prevent connection issues being flagged as failures
+                        ctx.summary['status'] = 'dead'
                     if 'failure_reason' not in ctx.summary:
                         ctx.summary['failure_reason'] = str(e)
                     log.exception('Manager failed: %s', taskname)