Thrasher: log backtrace of thrown exception

* add a wrapper to log uncaught exception to self.logger, greenlet also
  prints the backtrace and exception to stderr, but teuthology.log does
  not capture stderr. so we need to catch them by ourselves to reveal
  more info to root-cause this issue.
* log uncaught exception thrown by Thrasher.do_thrash() to self.log.

See: #10630
Signed-off-by: Kefu Chai <kchai@redhat.com>
(cherry picked from commit 64de3cd574b200d90cb3aea0d646a6236d8fec5e)

diff --git a/tasks/ceph_manager.py b/tasks/ceph_manager.py
index 45bbade02a006f1334503f8f40bf2ead4b52775f..71722232f71131cfea07f1617fe4eacfc91ba109 100644 (file)
--- a/tasks/ceph_manager.py
+++ b/tasks/ceph_manager.py
@@ -2,6 +2,7 @@
 ceph manager -- Thrasher and CephManager objects
 """
 from cStringIO import StringIO
+from functools import wraps
 import contextlib
 import random
 import time
@@ -10,6 +11,7 @@ import base64
 import json
 import logging
 import threading
+import traceback
 import os
 from teuthology import misc as teuthology
 from tasks.scrub import Scrubber
@@ -644,6 +646,17 @@ class Thrasher:
             val -= prob
         return None
 
+    def log_exc(func):
+        @wraps(func)
+        def wrapper(self):
+            try:
+                return func(self)
+            except:
+                self.log(traceback.format_exc())
+                raise
+        return wrapper
+
+    @log_exc
     def do_thrash(self):
         """
         Loop to select random actions to thrash ceph manager with.