qa/tasks/rook: add OSD creation to Rook QA

This commit adds OSD creation to the Rook QA tasks. The Rook task will
explicitly wait for the mgr to start and the CLI to work (instead of
implicitly doing so while waiting for 'ceph osd dump' to work).
Then it will do `ceph orch apply osd --all-available-devices` to create
OSDs on the rest of the PVs.

Signed-off-by: Joseph Sawaya <jsawaya@redhat.com>

diff --git a/qa/tasks/rook.py b/qa/tasks/rook.py
index df1dabf7945aa8bccc52cf73e36fbfa9bedc23ee..6d1920d23fda4735bc64ecd7c226439dae3db1f7 100644 (file)
--- a/qa/tasks/rook.py
+++ b/qa/tasks/rook.py
@@ -355,30 +355,6 @@ def rook_cluster(ctx, config):
                 'count': num_hosts,
                 'allowMultiplePerNode': True,
             },
-            'storage': {
-                'storageClassDeviceSets': [
-                    {
-                        'name': 'scratch',
-                        'count': num_devs,
-                        'portable': False,
-                        'volumeClaimTemplates': [
-                            {
-                                'metadata': {'name': 'data'},
-                                'spec': {
-                                    'resources': {
-                                        'requests': {
-                                            'storage': '10Gi'  # <= (lte) the actual PV size
-                                        }
-                                    },
-                                    'storageClassName': 'scratch',
-                                    'volumeMode': 'Block',
-                                    'accessModes': ['ReadWriteOnce'],
-                                },
-                            },
-                        ],
-                    }
-                ],
-            },
         }
     }
     teuthology.deep_merge(cluster['spec'], config.get('spec', {}))
@@ -464,11 +440,29 @@ def rook_toolbox(ctx, config):
         ], check_status=False)
 
 
+@contextlib.contextmanager
+def wait_for_orch(ctx, config):
+    log.info('Waiting for mgr/rook orchestrator to be available')
+    with safe_while(sleep=10, tries=90, action="check orch status") as proceed:
+        while proceed():
+            p = _shell(ctx, config, ['ceph', 'orch', 'status', '-f', 'json'],
+                       stdout=BytesIO(),
+                       check_status=False)
+            if p.exitstatus == 0:
+                r = json.loads(p.stdout.getvalue().decode('utf-8'))
+                if r.get('available') and r.get('backend') == 'rook':
+                    log.info(' mgr/rook orchestrator is active')
+                    break
+
+    yield
+
+
 @contextlib.contextmanager
 def rook_post_config(ctx, config):
     try:
         _shell(ctx, config, ['ceph', 'config', 'set', 'mgr', 'mgr/rook/storage_class',
                              'scratch'])
+        _shell(ctx, config, ['ceph', 'orch', 'apply', 'osd', '--all-available-devices'])
         yield
 
     except Exception as e:
@@ -630,8 +624,9 @@ def task(ctx, config):
             lambda: ceph_log(ctx, config),
             lambda: rook_cluster(ctx, config),
             lambda: rook_toolbox(ctx, config),
-            lambda: wait_for_osds(ctx, config),
+            lambda: wait_for_orch(ctx, config),
             lambda: rook_post_config(ctx, config),
+            lambda: wait_for_osds(ctx, config),
             lambda: ceph_config_keyring(ctx, config),
             lambda: ceph_clients(ctx, config),
     ):