From 79607eed3ca18c3a99cca9bf614aff8c56b8d462 Mon Sep 17 00:00:00 2001
From: Tommi Virtanen <tv@inktank.com>
Date: Tue, 11 Sep 2012 11:11:39 -0700
Subject: [PATCH] Don't lose tracebacks of exceptions raised in a greenlet.

Exception objects don't contain the traceback of where they were
raised from (to avoid cyclic data structures wrecking gc and causing
mem leaks), so the singular "raise obj" form creates a new traceback
from the current execution location, thus losing the original location
of the error.

Gevent explicitly wants to throw away the traceback, to release any
objects the greenlet may still be referring to, closing files,
releasing locks etc. In this case, we think it's safe, so stash the
exception info away in a holder object, and resurrect it on the other
side of the results queue.

http://stackoverflow.com/questions/9268916/how-to-capture-a-traceback-in-gevent

This can be reproduced easily with

	from teuthology.parallel import parallel
	def f():
	    raise RuntimeError("bork")
	with parallel() as p:
	    p.spawn(f)

and looking at the resulting traceback with and without this change.
---
 teuthology/parallel.py | 39 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/teuthology/parallel.py b/teuthology/parallel.py
index a3693ea6041cb..dbc69ba42a483 100644
--- a/teuthology/parallel.py
+++ b/teuthology/parallel.py
@@ -1,10 +1,36 @@
 import logging
+import sys
 
 import gevent.pool
 import gevent.queue
 
 log = logging.getLogger(__name__)
 
+class ExceptionHolder(object):
+    def __init__(self, exc_info):
+        self.exc_info = exc_info
+
+def capture_traceback(func, *args, **kwargs):
+    """
+    Utility function to capture tracebacks of any exception func
+    raises.
+    """
+    try:
+        return func(*args, **kwargs)
+    except Exception:
+        return ExceptionHolder(sys.exc_info())
+
+def resurrect_traceback(exc):
+    if isinstance(exc, ExceptionHolder):
+        exc_info = exc.exc_info
+    elif isinstance(exc, BaseException):
+        print type(exc)
+        exc_info = (type(exc), exc, None)
+    else:
+        return
+
+    raise exc_info[0], exc_info[1], exc_info[2]
+
 class parallel(object):
     """
     This class is a context manager for running functions in parallel.
@@ -41,7 +67,7 @@ class parallel(object):
     def spawn(self, func, *args, **kwargs):
         self.count += 1
         self.any_spawned = True
-        greenlet = self.group.spawn(func, *args, **kwargs)
+        greenlet = self.group.spawn(capture_traceback, func, *args, **kwargs)
         greenlet.link(self._finish)
 
     def __enter__(self):
@@ -69,10 +95,13 @@ class parallel(object):
         if not self.any_spawned or self.iteration_stopped:
             raise StopIteration()
         result = self.results.get()
-        if isinstance(result, BaseException):
-            if isinstance(result, StopIteration):
-                self.iteration_stopped = True
-            raise result
+
+        try:
+            resurrect_traceback(result)
+        except StopIteration:
+            self.iteration_stopped = True
+            raise
+
         return result
 
     def _finish(self, greenlet):
-- 
2.39.5