qa: enhance test cases

Fixes: https://tracker.ceph.com/issues/63105
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
(cherry picked from commit 9005451882371948359a1466fca10256476c5c37)

diff --git a/qa/tasks/cephfs/test_client_recovery.py b/qa/tasks/cephfs/test_client_recovery.py
index 61d23d596d0a6437cd25a34d1d0f6cd65b9e4c4d..082e7fa84d2b8bddc875dd7404584943c0a67338 100644 (file)
--- a/qa/tasks/cephfs/test_client_recovery.py
+++ b/qa/tasks/cephfs/test_client_recovery.py
@@ -13,6 +13,7 @@ import re
 import string
 import os
 
+from teuthology import contextutil
 from teuthology.orchestra import run
 from teuthology.orchestra.run import CommandFailedError
 from tasks.cephfs.fuse_mount import FuseMount
@@ -750,24 +751,27 @@ class TestClientOnLaggyOSD(CephFSTestCase):
             # it takes time to have laggy clients entries in cluster log,
             # wait for 6 minutes to see if it is visible, finally restart
             # the client
-            tries = 6
-            while True:
-                try:
-                    with self.assert_cluster_log("1 client(s) laggy due to laggy OSDs",
-                                                 timeout=55):
-                        # make sure clients weren't evicted
-                        self.assert_session_count(2)
-                        break
-                except AssertionError:
-                    tries -= 1
-                    if tries:
-                        continue
-                    raise
+            with contextutil.safe_while(sleep=5, tries=6) as proceed:
+                while proceed():
+                    try:
+                        with self.assert_cluster_log("1 client(s) laggy due to"
+                                                     " laggy OSDs",
+                                                     timeout=55):
+                            # make sure clients weren't evicted
+                            self.assert_session_count(2)
+                            break
+                    except (AssertionError, CommandFailedError) as e:
+                        log.debug(f'{e}, retrying')
+
+            # clear lagginess, expect to get the warning cleared and make sure
+            # client gets evicted
+            self.clear_laggy_params(osd)
+            self.wait_for_health_clear(60)
+            self.assert_session_count(1)
         finally:
             self.mount_a.kill_cleanup()
             self.mount_a.mount_wait()
             self.mount_a.create_destroy()
-            self.clear_laggy_params(osd)
 
     def test_client_eviction_if_config_is_unset(self):
         """
@@ -799,6 +803,11 @@ class TestClientOnLaggyOSD(CephFSTestCase):
 
             time.sleep(session_timeout)
             self.assert_session_count(1)
+
+            # make sure warning wasn't seen in cluster log
+            with self.assert_cluster_log("laggy due to laggy OSDs",
+                                         timeout=120, present=False):
+                pass
         finally:
             self.mount_a.kill_cleanup()
             self.mount_a.mount_wait()