Don't lose tracebacks of exceptions raised in a greenlet.

Exception objects don't contain the traceback of where they were
raised from (to avoid cyclic data structures wrecking gc and causing
mem leaks), so the singular "raise obj" form creates a new traceback
from the current execution location, thus losing the original location
of the error.

Gevent explicitly wants to throw away the traceback, to release any
objects the greenlet may still be referring to, closing files,
releasing locks etc. In this case, we think it's safe, so stash the
exception info away in a holder object, and resurrect it on the other
side of the results queue.

http://stackoverflow.com/questions/9268916/how-to-capture-a-traceback-in-gevent

This can be reproduced easily with

	from teuthology.parallel import parallel
	def f():
	    raise RuntimeError("bork")
	with parallel() as p:
	    p.spawn(f)

and looking at the resulting traceback with and without this change.

diff --git a/teuthology/parallel.py b/teuthology/parallel.py
index a3693ea6041cbcdbd38c2a271cd33ceea9beec01..dbc69ba42a48313c5953b3a1786d04751963a4f7 100644 (file)
--- a/teuthology/parallel.py
+++ b/teuthology/parallel.py
@@ -1,10 +1,36 @@
 import logging
+import sys
 
 import gevent.pool
 import gevent.queue
 
 log = logging.getLogger(__name__)
 
+class ExceptionHolder(object):
+    def __init__(self, exc_info):
+        self.exc_info = exc_info
+
+def capture_traceback(func, *args, **kwargs):
+    """
+    Utility function to capture tracebacks of any exception func
+    raises.
+    """
+    try:
+        return func(*args, **kwargs)
+    except Exception:
+        return ExceptionHolder(sys.exc_info())
+
+def resurrect_traceback(exc):
+    if isinstance(exc, ExceptionHolder):
+        exc_info = exc.exc_info
+    elif isinstance(exc, BaseException):
+        print type(exc)
+        exc_info = (type(exc), exc, None)
+    else:
+        return
+
+    raise exc_info[0], exc_info[1], exc_info[2]
+
 class parallel(object):
     """
     This class is a context manager for running functions in parallel.
@@ -41,7 +67,7 @@ class parallel(object):
     def spawn(self, func, *args, **kwargs):
         self.count += 1
         self.any_spawned = True
-        greenlet = self.group.spawn(func, *args, **kwargs)
+        greenlet = self.group.spawn(capture_traceback, func, *args, **kwargs)
         greenlet.link(self._finish)
 
     def __enter__(self):
@@ -69,10 +95,13 @@ class parallel(object):
         if not self.any_spawned or self.iteration_stopped:
             raise StopIteration()
         result = self.results.get()
-        if isinstance(result, BaseException):
-            if isinstance(result, StopIteration):
-                self.iteration_stopped = True
-            raise result
+
+        try:
+            resurrect_traceback(result)
+        except StopIteration:
+            self.iteration_stopped = True
+            raise
+
         return result
 
     def _finish(self, greenlet):