thrashosds: fail if cluster doesn't finally become clean in 5 minutes

diff --git a/teuthology/task/ceph_manager.py b/teuthology/task/ceph_manager.py
index 194a9f49a71d7675fb606ea932772946615d4120..da696a59c8ec20f7ee57c2d830a20fa6e45ffc25 100644 (file)
--- a/teuthology/task/ceph_manager.py
+++ b/teuthology/task/ceph_manager.py
@@ -158,9 +158,13 @@ class CephManager:
     def is_clean(self):
         return self.get_num_active_clean() == self.get_num_pgs()
 
-    def wait_till_clean(self):
+    def wait_till_clean(self, timeout=None):
         self.log("waiting till clean")
+        start = time.time()
         while not self.is_clean():
+            if timeout is not None:
+                assert time.time() - start < timeout, \
+                    'failed to become clean before timeout expired'
             time.sleep(3)
         self.log("clean!")
 

diff --git a/teuthology/task/thrashosds.py b/teuthology/task/thrashosds.py
index a941718077c0b9c8beea8fdbf5a0a8de0b240398..6e135c19b0087501fc47199245945c57b361cdac 100644 (file)
--- a/teuthology/task/thrashosds.py
+++ b/teuthology/task/thrashosds.py
@@ -39,7 +39,11 @@ def task(ctx, config):
        wrongly marked down will mark itself back up again.) This value
        can be either an integer (eg, 75) or a float probability (eg
        0.75).
-    
+
+    timeout: (360) the number of seconds to wait for the cluster
+       to become clean before the task exits. If this doesn't happen,
+       an exception will be raised.
+
     example:
 
     tasks:
@@ -48,6 +52,7 @@ def task(ctx, config):
         chance_down: 10
         op_delay: 3
         min_in: 1
+        timeout: 600
     - interactive:
     """
     log.info('Beginning thrashosds...')
@@ -67,4 +72,4 @@ def task(ctx, config):
     finally:
         log.info('joining thrashosds')
         thrash_proc.do_join()
-        manager.wait_till_clean()
+        manager.wait_till_clean(config.get('timeout', 360))