ceph_manager: reset timeout if we are making progress in wait_for_recovery

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/teuthology/task/ceph_manager.py b/teuthology/task/ceph_manager.py
index 046ed644b5984eaf934d939a92a62f983d6e9606..c059ec46f5a46d05458c84e172edb9d5a6468d0a 100644 (file)
--- a/teuthology/task/ceph_manager.py
+++ b/teuthology/task/ceph_manager.py
@@ -1119,8 +1119,13 @@ class CephManager:
         num_active_recovered = self.get_num_active_recovered()
         while not self.is_recovered():
             if timeout is not None:
-                assert time.time() - start < timeout, \
-                    'failed to recover before timeout expired'
+                if self.get_is_making_recovery_progress():
+                    self.log("making progress, resetting timeout")
+                    start = time.time()
+                else:
+                    self.log("no progress seen, keeping timeout for now")
+                    assert time.time() - start < timeout, \
+                        'failed to recover before timeout expired'
             cur_active_recovered = self.get_num_active_recovered()
             if cur_active_recovered != num_active_recovered:
                 start = time.time()