From 08ace5465e08925054def627508247eb2d752041 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 21 May 2014 11:34:11 -0500 Subject: [PATCH] Add exception hook to teuthology-worker Workers processes are dying occasionally, and this should cause the exceptions to be logged. Signed-off-by: Zack Cerza --- teuthology/worker.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/teuthology/worker.py b/teuthology/worker.py index f1ca4248dd..e74167c069 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -9,6 +9,7 @@ import time import yaml from datetime import datetime +from traceback import format_tb from teuthology import setup_log_file from . import beanstalk @@ -40,6 +41,23 @@ def restart(): os.execv(sys.executable, args) +def install_except_hook(): + """ + Install an exception hook that first logs any uncaught exception, then + raises it. + """ + def log_exception(exception_class, exception, traceback): + logging.critical(''.join(format_tb(traceback))) + if not exception.message: + logging.critical(exception_class.__name__) + else: + logging.critical('{0}: {1}'.format( + exception_class.__name__, exception)) + # Now raise the exception like normal + sys.__excepthook__(exception_class, exception, traceback) + sys.excepthook = log_exception + + class filelock(object): # simple flock class def __init__(self, fn): @@ -129,6 +147,8 @@ def main(ctx): pid=os.getpid(), tube=ctx.tube,)) setup_log_file(log, log_file_path) + install_except_hook() + if not os.path.isdir(ctx.archive_dir): sys.exit("{prog}: archive directory must exist: {path}".format( prog=os.path.basename(sys.argv[0]), -- 2.39.5