From 0be5f1f015874477993b558464f83972bc0e1aa9 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Fri, 7 Mar 2014 11:21:31 -0600 Subject: [PATCH] Introduce a timeout to radosbench's join phase Set to 15min right now. Signed-off-by: Zack Cerza --- teuthology/orchestra/run.py | 17 +++++++++++++++-- teuthology/task/radosbench.py | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/teuthology/orchestra/run.py b/teuthology/orchestra/run.py index 16cb07bfcf..d00bc63643 100644 --- a/teuthology/orchestra/run.py +++ b/teuthology/orchestra/run.py @@ -9,6 +9,8 @@ import pipes import logging import shutil +from ..contextutil import safe_while + log = logging.getLogger(__name__) class RemoteProcess(object): @@ -214,7 +216,7 @@ def spawn_asyncresult(fn, *args, **kwargs): class Sentinel(object): """ - Sentinel -- used to define PIPE file-like object. + Sentinel -- used to define PIPE file-like object. """ def __init__(self, name): self.name = name @@ -332,12 +334,23 @@ def run( return r -def wait(processes): +def wait(processes, timeout=None): """ Wait for all given processes to exit. Raise if any one of them fails. + + Optionally, timeout after 'timeout' seconds. """ + if timeout and timeout > 0: + with safe_while(sleep=5, increment=0, tries=(timeout/5)) as check_time: + not_ready = list(processes) + while len(not_ready) > 0: + check_time() + for proc in not_ready: + if proc.ready(): + not_ready.remove(proc) + for proc in processes: assert isinstance(proc.exitstatus, gevent.event.AsyncResult) proc.exitstatus.get() diff --git a/teuthology/task/radosbench.py b/teuthology/task/radosbench.py index f8248aca13..8b72fc38e2 100644 --- a/teuthology/task/radosbench.py +++ b/teuthology/task/radosbench.py @@ -76,7 +76,7 @@ def task(ctx, config): yield finally: log.info('joining radosbench') - run.wait(radosbench.itervalues()) + run.wait(radosbench.itervalues(), timeout=900) if pool is not 'data': ctx.manager.remove_pool(pool) -- 2.39.5