qa/cephadm: ensure host has been fully saved before considering bootstrap complete

Fixes: https://tracker.ceph.com/issues/74978
Signed-off-by: Adam King <adking@redhat.com>

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 2fbe527e3029edc4b0d1df565287ba928aa29cd2..4b52a7b3f942296d62a9dc30fea8ee02134a577c 100644 (file)
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -804,11 +804,29 @@ def ceph_bootstrap(ctx, config):
                 'ceph', 'orch', 'host', 'add',
                 remote.shortname
             ])
-            r = _shell(ctx, cluster_name, bootstrap_remote,
-                       ['ceph', 'orch', 'host', 'ls', '--format=json'],
-                       stdout=StringIO())
-            hosts = [node['hostname'] for node in json.loads(r.stdout.getvalue())]
-            assert remote.shortname in hosts
+            with contextutil.safe_while(sleep=5, tries=3) as proceed:
+                while proceed():
+                    # check host has been added
+                    r = _shell(ctx, cluster_name, bootstrap_remote,
+                               ['ceph', 'orch', 'host', 'ls', '--format=json'],
+                               stdout=StringIO())
+                    hosts = [node['hostname'] for node in json.loads(r.stdout.getvalue())]
+                    # check host has been given config-key store entry
+                    r = _shell(ctx, cluster_name, bootstrap_remote,
+                               ['ceph', 'config-key', 'ls'],
+                               stdout=StringIO())
+                    key_entries = r.stdout.getvalue()
+                    # check host has been added to config-key inventory entry
+                    r = _shell(ctx, cluster_name, bootstrap_remote,
+                               ['ceph', 'config-key', 'get', 'mgr/cephadm/inventory'],
+                               stdout=StringIO())
+                    stored_inventory = json.loads(r.stdout.getvalue())
+                    if (
+                        remote.shortname in hosts
+                        and  remote.shortname in key_entries
+                        and remote.shortname in stored_inventory
+                    ):
+                        break
 
         yield