rados: check that pool is done trimming before removing it

Signed-off-by: Samuel Just <sjust@redhat.com>
(cherry picked from commit 4aebf59d906fa3e03d21bdac182f89fe3cd4c802)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index d4a23a00c5d49fb5c6fcbd39fb1dcee062abbb94..87e7edbc13f147f8532e289482623289fa42bd0b 100644 (file)
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -1624,6 +1624,33 @@ class CephManager:
             time.sleep(SLEEP_TIME)
             timer += SLEEP_TIME
 
+    def wait_snap_trimming_complete(self, pool):
+        """
+        Wait for snap trimming on pool to end
+        """
+        POLL_PERIOD = 10
+        FATAL_TIMEOUT = 600
+        start = time.time()
+        poolnum = self.get_pool_num(pool)
+        poolnumstr = "%s." % (poolnum,)
+        while (True):
+            now = time.time()
+            if (now - start) > FATAL_TIMEOUT:
+                assert (now - start) < FATAL_TIMEOUT, \
+                    'failed to complete snap trimming before timeout'
+            all_stats = self.get_pg_stats()
+            trimming = False
+            for pg in all_stats:
+                if (poolnumstr in pg['pgid']) and ('snaptrim' in pg['state']):
+                    self.log("pg {pg} in trimming, state: {state}".format(
+                        pg=pg['pgid'],
+                        state=pg['state']))
+                    trimming = True
+            if not trimming:
+                break
+            self.log("{pool} still trimming, waiting".format(pool=pool))
+            time.sleep(POLL_PERIOD)
+
     def get_single_pg_stats(self, pgid):
         """
         Return pg for the pgid specified.

diff --git a/qa/tasks/rados.py b/qa/tasks/rados.py
index ad100667aea7660ad8e6178fe3bff5b53c9dfe52..15a35e5988ddc46bf284254533085ec91c9a44bc 100644 (file)
--- a/qa/tasks/rados.py
+++ b/qa/tasks/rados.py
@@ -250,6 +250,7 @@ def task(ctx, config):
             run.wait(tests.itervalues())
 
             for pool in created_pools:
+                manager.wait_snap_trimming_complete(pool);
                 manager.remove_pool(pool)
 
     running = gevent.spawn(thread)