From 387c4f13100ebb24fe438d05d3c6820434f409dd Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Tue, 21 Sep 2021 09:41:28 -0400 Subject: [PATCH] qa/tasks/rook: fix cluster deletion hanging due to CephObjectStore CR This commit fixes the issue where the cluster deletion hangs in the QA while a CephObjectStore CR is still up by removing all rgw/nfs/mds/rbd-mirror daemons before tearing down the rest of the cluster. Signed-off-by: Joseph Sawaya --- qa/tasks/rook.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/qa/tasks/rook.py b/qa/tasks/rook.py index b005191ccc920..f197b0f472da7 100644 --- a/qa/tasks/rook.py +++ b/qa/tasks/rook.py @@ -646,4 +646,28 @@ def task(ctx, config): yield finally: + to_remove = [] + ret = _shell(ctx, config, ['ceph', 'orch', 'ls', '-f', 'json'], stdout=BytesIO()) + if ret.exitstatus == 0: + r = json.loads(ret.stdout.getvalue().decode('utf-8')) + for service in r: + removal_name = None + if service['service_type'] == 'rgw': + removal_name = 'rgw.' + service['spec']['rgw_realm'] + elif service['service_type'] == 'mds': + removal_name = service['service_name'] + elif service['service_type'] == 'nfs': + removal_name = service['service_name'] + if removal_name != None: + _shell(ctx, config, ['ceph', 'orch', 'rm', removal_name]) + to_remove.append(service['service_name']) + with safe_while(sleep=10, tries=90, action="waiting for service removal") as proceed: + while proceed(): + ret = _shell(ctx, config, ['ceph', 'orch', 'ls', '-f', 'json'], stdout=BytesIO()) + if ret.exitstatus == 0: + r = json.loads(ret.stdout.getvalue().decode('utf-8')) + still_up = [service['service_name'] for service in r] + matches = set(still_up).intersection(to_remove) + if not matches: + break log.info('Tearing down rook') -- 2.39.5