qa/tasks/rook: more orderly cluster teardown

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/qa/tasks/rook.py b/qa/tasks/rook.py
index 0230f2778bff5017b97bdcc3fc61a3797efd248c..ef21a5b880b4d9c654eceb9ff6730b841a41d52d 100644 (file)
--- a/qa/tasks/rook.py
+++ b/qa/tasks/rook.py
@@ -277,8 +277,34 @@ def rook_cluster(ctx, config):
 
     finally:
         _kubectl(ctx, config, ['delete', '-f', 'cluster.yaml'], check_status=False)
-        _kubectl(ctx, config, ['delete', 'configmap', 'rook-config-override'],
-                 check_status=False)
+
+        # wait for cluster to shut down
+        log.info('Waiting for cluster to stop')
+        running = True
+        with safe_while(sleep=5, tries=100, action="wait for teardown") as proceed:
+            while running and proceed():
+                p = _kubectl(
+                    ctx, config,
+                    ['-n', 'rook-ceph', 'get', 'pods'],
+                    stdout=BytesIO(),
+                )
+                running = False
+                for line in p.stdout.getvalue().decode('utf-8').strip().splitlines():
+                    name, ready, status, _ = line.split(None, 3)
+                    if (
+                            name != 'NAME'
+                            and not name.startswith('csi-')
+                            and not name.startswith('rook-ceph-operator-')
+                            and not name.startswith('rook-ceph-tools-')
+                    ):
+                        running = True
+                        break
+
+        _kubectl(
+            ctx, config,
+            ['-n', 'rook-ceph', 'delete', 'configmap', 'rook-config-override'],
+            check_status=False,
+        )
         ctx.rook[cluster_name].remote.run(args=['rm', '-f', 'cluster.yaml'])