Add exception hook to teuthology-worker

Workers processes are dying occasionally, and this should cause the
exceptions to be logged.

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/teuthology/worker.py b/teuthology/worker.py
index f1ca4248dda5fb706199199c05da4252ce2c39fb..e74167c069036ad553a084ffab5a190a76df7e76 100644 (file)
--- a/teuthology/worker.py
+++ b/teuthology/worker.py
@@ -9,6 +9,7 @@ import time
 import yaml
 
 from datetime import datetime
+from traceback import format_tb
 
 from teuthology import setup_log_file
 from . import beanstalk
@@ -40,6 +41,23 @@ def restart():
     os.execv(sys.executable, args)
 
 
+def install_except_hook():
+    """
+    Install an exception hook that first logs any uncaught exception, then
+    raises it.
+    """
+    def log_exception(exception_class, exception, traceback):
+        logging.critical(''.join(format_tb(traceback)))
+        if not exception.message:
+            logging.critical(exception_class.__name__)
+        else:
+            logging.critical('{0}: {1}'.format(
+                exception_class.__name__, exception))
+        # Now raise the exception like normal
+        sys.__excepthook__(exception_class, exception, traceback)
+    sys.excepthook = log_exception
+
+
 class filelock(object):
     # simple flock class
     def __init__(self, fn):
@@ -129,6 +147,8 @@ def main(ctx):
         pid=os.getpid(), tube=ctx.tube,))
     setup_log_file(log, log_file_path)
 
+    install_except_hook()
+
     if not os.path.isdir(ctx.archive_dir):
         sys.exit("{prog}: archive directory must exist: {path}".format(
             prog=os.path.basename(sys.argv[0]),