qa: timeout waiting for thrashed MDS to revive

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/qa/tasks/mds_thrash.py b/qa/tasks/mds_thrash.py
index cda13d14592721b6f9895dc3a29d0c9abfb8f123..e6e5e08065f0e432be9f1b010927ebdc229c64e6 100644 (file)
--- a/qa/tasks/mds_thrash.py
+++ b/qa/tasks/mds_thrash.py
@@ -4,6 +4,7 @@ Thrash mds by simulating failures
 import logging
 import contextlib
 import ceph_manager
+import itertools
 import random
 import time
 
@@ -172,8 +173,7 @@ class MDSThrasher(Greenlet):
 
     def wait_for_stable(self, rank = None, gid = None):
         self.log('waiting for mds cluster to stabilize...')
-        itercount = 0
-        while True:
+        for itercount in itertools.count():
             status = self.fs.status()
             max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds']
             ranks = list(status.get_ranks(self.fs.id))
@@ -203,7 +203,6 @@ class MDSThrasher(Greenlet):
                  raise RuntimeError('timeout waiting for cluster to stabilize')
             elif itercount % 5 == 0:
                 self.log('mds map: {status}'.format(status=self.fs.status()))
-            itercount = itercount + 1
             time.sleep(2)
 
     def do_thrash(self):
@@ -318,7 +317,9 @@ class MDSThrasher(Greenlet):
                 self.log('reviving {label}'.format(label=label))
                 self.revive_mds(name)
 
-                while True:
+                for itercount in itertools.count():
+                    if itercount > 300/2: # 5 minutes
+                        raise RuntimeError('timeout waiting for MDS to revive')
                     status = self.fs.status()
                     info = status.get_mds(name)
                     if info and info['state'] in ('up:standby', 'up:standby-replay'):