Introduce a timeout to radosbench's join phase

Set to 15min right now.

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/teuthology/orchestra/run.py b/teuthology/orchestra/run.py
index 16cb07bfcf8eeab41dbbcdf4014179ff6c8e63b4..d00bc6364397cde0623717d6d90a13000d6e7b32 100644 (file)
--- a/teuthology/orchestra/run.py
+++ b/teuthology/orchestra/run.py
@@ -9,6 +9,8 @@ import pipes
 import logging
 import shutil
 
+from ..contextutil import safe_while
+
 log = logging.getLogger(__name__)
 
 class RemoteProcess(object):
@@ -214,7 +216,7 @@ def spawn_asyncresult(fn, *args, **kwargs):
 
 class Sentinel(object):
     """
-    Sentinel -- used to define PIPE file-like object. 
+    Sentinel -- used to define PIPE file-like object.
     """
     def __init__(self, name):
         self.name = name
@@ -332,12 +334,23 @@ def run(
     return r
 
 
-def wait(processes):
+def wait(processes, timeout=None):
     """
     Wait for all given processes to exit.
 
     Raise if any one of them fails.
+
+    Optionally, timeout after 'timeout' seconds.
     """
+    if timeout and timeout > 0:
+        with safe_while(sleep=5, increment=0, tries=(timeout/5)) as check_time:
+            not_ready = list(processes)
+            while len(not_ready) > 0:
+                check_time()
+                for proc in not_ready:
+                    if proc.ready():
+                        not_ready.remove(proc)
+
     for proc in processes:
         assert isinstance(proc.exitstatus, gevent.event.AsyncResult)
         proc.exitstatus.get()

diff --git a/teuthology/task/radosbench.py b/teuthology/task/radosbench.py
index f8248aca130edfe25a83b4d528d13de32c437e9e..8b72fc38e29cd09914b364e6d701addad791ce21 100644 (file)
--- a/teuthology/task/radosbench.py
+++ b/teuthology/task/radosbench.py
@@ -76,7 +76,7 @@ def task(ctx, config):
         yield
     finally:
         log.info('joining radosbench')
-        run.wait(radosbench.itervalues())
+        run.wait(radosbench.itervalues(), timeout=900)
 
         if pool is not 'data':
             ctx.manager.remove_pool(pool)