From: Joseph Sawaya <jsawaya@redhat.com>
Date: Tue, 21 Sep 2021 13:41:28 +0000 (-0400)
Subject: qa/tasks/rook: fix cluster deletion hanging due to CephObjectStore CR
X-Git-Tag: v17.1.0~781^2~3
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=387c4f13100ebb24fe438d05d3c6820434f409dd;p=ceph-ci.git

qa/tasks/rook: fix cluster deletion hanging due to CephObjectStore CR

This commit fixes the issue where the cluster deletion hangs in the QA
while a CephObjectStore CR is still up by removing all rgw/nfs/mds/rbd-mirror
daemons before tearing down the rest of the cluster.

Signed-off-by: Joseph Sawaya <jsawaya@redhat.com>
---

diff --git a/qa/tasks/rook.py b/qa/tasks/rook.py
index b005191ccc9..f197b0f472d 100644
--- a/qa/tasks/rook.py
+++ b/qa/tasks/rook.py
@@ -646,4 +646,28 @@ def task(ctx, config):
             yield
 
         finally:
+            to_remove = []
+            ret = _shell(ctx, config, ['ceph', 'orch', 'ls', '-f', 'json'], stdout=BytesIO())
+            if ret.exitstatus == 0:
+                r = json.loads(ret.stdout.getvalue().decode('utf-8'))
+                for service in r:
+                    removal_name = None
+                    if service['service_type'] == 'rgw':
+                        removal_name = 'rgw.' + service['spec']['rgw_realm']
+                    elif service['service_type'] == 'mds':
+                        removal_name = service['service_name']
+                    elif service['service_type'] == 'nfs':
+                        removal_name = service['service_name']
+                    if removal_name != None:
+                        _shell(ctx, config, ['ceph', 'orch', 'rm', removal_name])
+                        to_remove.append(service['service_name'])
+                with safe_while(sleep=10, tries=90, action="waiting for service removal") as proceed:
+                    while proceed():
+                        ret = _shell(ctx, config, ['ceph', 'orch', 'ls', '-f', 'json'], stdout=BytesIO())
+                        if ret.exitstatus == 0:
+                            r = json.loads(ret.stdout.getvalue().decode('utf-8'))
+                            still_up = [service['service_name'] for service in r]
+                            matches = set(still_up).intersection(to_remove)
+                            if not matches:
+                                break
             log.info('Tearing down rook')