qa: wait for MDS to come back after removing it

Fixes: http://tracker.ceph.com/issues/40967
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/qa/tasks/cephfs/test_failover.py b/qa/tasks/cephfs/test_failover.py
index 527386006fb995c93831d90726cfd4ea01ccaa1d..786980ba08763a559d1eae4faa161257c4d10064 100644 (file)
--- a/qa/tasks/cephfs/test_failover.py
+++ b/qa/tasks/cephfs/test_failover.py
@@ -364,7 +364,7 @@ class TestStandbyReplay(CephFSTestCase):
         self.assertEqual(0, len(list(self.fs.get_replays(status=status))))
         return status
 
-    def _confirm_single_replay(self, full=True, status=None):
+    def _confirm_single_replay(self, full=True, status=None, retries=3):
         status = self.fs.wait_for_daemons(status=status)
         ranks = sorted(self.fs.get_mds_map(status=status)['in'])
         replays = list(self.fs.get_replays(status=status))
@@ -377,7 +377,11 @@ class TestStandbyReplay(CephFSTestCase):
                     has_replay = True
                     checked_replays.add(replay['gid'])
             if full and not has_replay:
-                raise RuntimeError("rank "+str(rank)+" has no standby-replay follower")
+                if retries <= 0:
+                    raise RuntimeError("rank "+str(rank)+" has no standby-replay follower")
+                else:
+                    retries = retries-1
+                    time.sleep(2)
         self.assertEqual(checked_replays, set(info['gid'] for info in replays))
         return status