From 8dd52f994164d9a99efaa17a3051ec5a9f7d061e Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Thu, 8 Sep 2011 18:09:11 -0700 Subject: [PATCH] thrashosds: fail if cluster doesn't finally become clean in 5 minutes --- teuthology/task/ceph_manager.py | 6 +++++- teuthology/task/thrashosds.py | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/teuthology/task/ceph_manager.py b/teuthology/task/ceph_manager.py index 194a9f49a71d7..da696a59c8ec2 100644 --- a/teuthology/task/ceph_manager.py +++ b/teuthology/task/ceph_manager.py @@ -158,9 +158,13 @@ class CephManager: def is_clean(self): return self.get_num_active_clean() == self.get_num_pgs() - def wait_till_clean(self): + def wait_till_clean(self, timeout=None): self.log("waiting till clean") + start = time.time() while not self.is_clean(): + if timeout is not None: + assert time.time() - start < timeout, \ + 'failed to become clean before timeout expired' time.sleep(3) self.log("clean!") diff --git a/teuthology/task/thrashosds.py b/teuthology/task/thrashosds.py index a941718077c0b..6e135c19b0087 100644 --- a/teuthology/task/thrashosds.py +++ b/teuthology/task/thrashosds.py @@ -39,7 +39,11 @@ def task(ctx, config): wrongly marked down will mark itself back up again.) This value can be either an integer (eg, 75) or a float probability (eg 0.75). - + + timeout: (360) the number of seconds to wait for the cluster + to become clean before the task exits. If this doesn't happen, + an exception will be raised. + example: tasks: @@ -48,6 +52,7 @@ def task(ctx, config): chance_down: 10 op_delay: 3 min_in: 1 + timeout: 600 - interactive: """ log.info('Beginning thrashosds...') @@ -67,4 +72,4 @@ def task(ctx, config): finally: log.info('joining thrashosds') thrash_proc.do_join() - manager.wait_till_clean() + manager.wait_till_clean(config.get('timeout', 360)) -- 2.39.5