qa/tasks/ceph_manager: add a "wait_for_pg_stats()" decorator

and accompany it with two helpers to access the pg stats in a more
natural way

Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 8ff2556a7a0e9334c296c44f25b75063690dd049..2d963e2cc3e24ede11022a835fdedebce88dea97 100644 (file)
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -1305,6 +1305,23 @@ class CephManager:
                 return int(pg['acting'][-1])
         assert False
 
+    def wait_for_pg_stats(func):
+        # both osd_mon_report_interval_min and mgr_stats_period are 5 seconds
+        # by default, and take the faulty injection in ms into consideration,
+        # 12 seconds are more than enough
+        delays = [1, 1, 2, 3, 5, 8, 13]
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            exc = None
+            for delay in delays:
+                try:
+                    return func(self, *args, **kwargs)
+                except AssertionError as e:
+                    time.sleep(delay)
+                    exc = e
+            raise exc
+        return wrapper
+
     def get_pg_primary(self, pool, pgnum):
         """
         get primary for pool, pgnum (e.g. (data, 0)->0
@@ -1723,6 +1740,18 @@ class CephManager:
         stats = self.get_single_pg_stats(pgstr)
         return 'inconsistent' in stats['state']
 
+    @wait_for_pg_stats
+    def with_pg_state(self, pool, pgnum, check):
+        pgstr = self.get_pgid(pool, pgnum)
+        stats = self.get_single_pg_stats(pgstr)
+        assert(check(stats['state']))
+
+    @wait_for_pg_stats
+    def with_pg(self, pool, pgnum, check):
+        pgstr = self.get_pgid(pool, pgnum)
+        stats = self.get_single_pg_stats(pgstr)
+        return check(stats)
+
     def get_last_scrub_stamp(self, pool, pgnum):
         """
         Get the timestamp of the last scrub.