qa/tasks/rook: fix cluster deletion hanging due to CephObjectStore CR

This commit fixes the issue where the cluster deletion hangs in the QA
while a CephObjectStore CR is still up by removing all rgw/nfs/mds/rbd-mirror
daemons before tearing down the rest of the cluster.

Signed-off-by: Joseph Sawaya <jsawaya@redhat.com>

diff --git a/qa/tasks/rook.py b/qa/tasks/rook.py
index b005191ccc92099afe72e04f9f5b3ffacc53985f..f197b0f472da7f355ab9746478bc158f36fcefd1 100644 (file)
--- a/qa/tasks/rook.py
+++ b/qa/tasks/rook.py
@@ -646,4 +646,28 @@ def task(ctx, config):
             yield
 
         finally:
+            to_remove = []
+            ret = _shell(ctx, config, ['ceph', 'orch', 'ls', '-f', 'json'], stdout=BytesIO())
+            if ret.exitstatus == 0:
+                r = json.loads(ret.stdout.getvalue().decode('utf-8'))
+                for service in r:
+                    removal_name = None
+                    if service['service_type'] == 'rgw':
+                        removal_name = 'rgw.' + service['spec']['rgw_realm']
+                    elif service['service_type'] == 'mds':
+                        removal_name = service['service_name']
+                    elif service['service_type'] == 'nfs':
+                        removal_name = service['service_name']
+                    if removal_name != None:
+                        _shell(ctx, config, ['ceph', 'orch', 'rm', removal_name])
+                        to_remove.append(service['service_name'])
+                with safe_while(sleep=10, tries=90, action="waiting for service removal") as proceed:
+                    while proceed():
+                        ret = _shell(ctx, config, ['ceph', 'orch', 'ls', '-f', 'json'], stdout=BytesIO())
+                        if ret.exitstatus == 0:
+                            r = json.loads(ret.stdout.getvalue().decode('utf-8'))
+                            still_up = [service['service_name'] for service in r]
+                            matches = set(still_up).intersection(to_remove)
+                            if not matches:
+                                break
             log.info('Tearing down rook')