mgr/cephadm: identify and instantiate raw osds post-create

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py
index 775d296911ffa5d6d707b72d63bf5358475f362e..4d41891a9fb202b9b7727476cf46cee3548c9e28 100644 (file)
--- a/src/pybind/mgr/cephadm/services/osd.py
+++ b/src/pybind/mgr/cephadm/services/osd.py
@@ -98,7 +98,8 @@ class OSDService(CephService):
         if replace_osd_ids is None:
             replace_osd_ids = OsdIdClaims(self.mgr).filtered_by_host(host)
             assert replace_osd_ids is not None
-        # check result
+
+        # check result: lvm
         osds_elems: dict = CephadmServe(self.mgr)._run_cephadm_json(
             host, 'osd', 'ceph-volume',
             [
@@ -142,6 +143,43 @@ class OSDService(CephService):
                     daemon_spec,
                     osd_uuid_map=osd_uuid_map)
 
+        # check result: raw
+        raw_elems: dict = CephadmServe(self.mgr)._run_cephadm_json(
+            host, 'osd', 'ceph-volume',
+            [
+                '--',
+                'raw', 'list',
+                '--format', 'json',
+            ])
+        for osd_uuid, osd in raw_elems.items():
+            if osd.get('ceph_fsid') != fsid:
+                continue
+            osd_id = str(osd.get('osd_id', '-1'))
+            if osd_id in before_osd_uuid_map and osd_id not in replace_osd_ids:
+                # if it exists but is part of the replacement operation, don't skip
+                continue
+            if osd_id not in osd_uuid_map:
+                logger.debug('osd id {} does not exist in cluster'.format(osd_id))
+                continue
+            if osd_uuid_map.get(osd_id) != osd_uuid:
+                logger.debug('mismatched osd uuid (cluster has %s, osd '
+                             'has %s)' % (osd_uuid_map.get(osd_id), osd_uuid))
+                continue
+            if osd_id in created:
+                continue
+
+            created.append(osd_id)
+            daemon_spec = CephadmDaemonDeploySpec(
+                service_name=service_name,
+                daemon_id=osd_id,
+                host=host,
+                daemon_type='osd',
+            )
+            daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
+            CephadmServe(self.mgr)._create_daemon(
+                daemon_spec,
+                osd_uuid_map=osd_uuid_map)
+
         if created:
             self.mgr.cache.invalidate_host_devices(host)
             self.mgr.cache.invalidate_autotune(host)

diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py
index f93ededbdee6ee6c460abf07d6d9fb787af2f165..40e91daa91c247a8640706f1886de3c96bdf0392 100644 (file)
--- a/src/pybind/mgr/cephadm/tests/test_cephadm.py
+++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py
@@ -1,4 +1,5 @@
 import json
+import logging
 
 from contextlib import contextmanager
 
@@ -592,8 +593,10 @@ class TestCephadm(object):
                 ['--config-json', '-', '--', 'lvm', 'batch',
                     '--no-auto', '/dev/sdb', '--yes', '--no-systemd'],
                 env_vars=['CEPH_VOLUME_OSDSPEC_AFFINITY=foo'], error_ok=True, stdin='{"config": "", "keyring": ""}')
-            _run_cephadm.assert_called_with(
+            _run_cephadm.assert_any_call(
                 'test', 'osd', 'ceph-volume', ['--', 'lvm', 'list', '--format', 'json'], image='', no_fsid=False)
+            _run_cephadm.assert_any_call(
+                'test', 'osd', 'ceph-volume', ['--', 'raw', 'list', '--format', 'json'], image='', no_fsid=False)
 
     @mock.patch("cephadm.serve.CephadmServe._run_cephadm")
     def test_apply_osd_save_non_collocated(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
@@ -632,8 +635,10 @@ class TestCephadm(object):
                     '--wal-devices', '/dev/sdd', '--yes', '--no-systemd'],
                 env_vars=['CEPH_VOLUME_OSDSPEC_AFFINITY=noncollocated'],
                 error_ok=True, stdin='{"config": "", "keyring": ""}')
-            _run_cephadm.assert_called_with(
+            _run_cephadm.assert_any_call(
                 'test', 'osd', 'ceph-volume', ['--', 'lvm', 'list', '--format', 'json'], image='', no_fsid=False)
+            _run_cephadm.assert_any_call(
+                'test', 'osd', 'ceph-volume', ['--', 'raw', 'list', '--format', 'json'], image='', no_fsid=False)
 
     @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}'))
     @mock.patch("cephadm.module.SpecStore.save")
@@ -1410,17 +1415,31 @@ Traceback (most recent call last):
                 ]
             })
 
-            ceph_volume_lvm_list = {
-                '1': [{
-                    'tags': {
-                        'ceph.cluster_fsid': cephadm_module._cluster_fsid,
-                        'ceph.osd_fsid': 'uuid'
-                    },
-                    'type': 'data'
-                }]
-            }
-            _run_cephadm.return_value = (json.dumps(ceph_volume_lvm_list), '', 0)
-            _run_cephadm.reset_mock()
+            def _ceph_volume_list(s, host, entity, cmd, **kwargs):
+                logging.info(f'ceph-volume cmd: {cmd}')
+                if 'raw' in cmd:
+                    return json.dumps({
+                        "21a4209b-f51b-4225-81dc-d2dca5b8b2f5": {
+                            "ceph_fsid": "64c84f19-fe1d-452a-a731-ab19dc144aa8",
+                            "device": "/dev/loop0",
+                            "osd_id": 21,
+                            "osd_uuid": "21a4209b-f51b-4225-81dc-d2dca5b8b2f5",
+                            "type": "bluestore"
+                        },
+                    }), '', 0
+                if 'lvm' in cmd:
+                    return json.dumps({
+                        '1': [{
+                            'tags': {
+                                'ceph.cluster_fsid': cephadm_module._cluster_fsid,
+                                'ceph.osd_fsid': 'uuid'
+                            },
+                            'type': 'data'
+                        }]
+                    }), '', 0
+                return '{}', '', 0
+            _run_cephadm.reset_mock(return_value=True)
+            _run_cephadm.side_effect = _ceph_volume_list
             assert cephadm_module._osd_activate(
                 ['test']).stdout == "Created osd(s) 1 on host 'test'"
             assert _run_cephadm.mock_calls == [
@@ -1430,6 +1449,8 @@ Traceback (most recent call last):
                           ['--name', 'osd.1', '--meta-json', mock.ANY,
                               '--config-json', '-', '--osd-fsid', 'uuid'],
                           stdin=mock.ANY, image=''),
+                mock.call('test', 'osd', 'ceph-volume',
+                          ['--', 'raw', 'list', '--format', 'json'], no_fsid=False, image=''),
             ]
 
     @mock.patch("cephadm.serve.CephadmServe._run_cephadm")
@@ -1486,23 +1507,37 @@ Traceback (most recent call last):
                 ]
             })
 
-            ceph_volume_lvm_list = {
-                '1': [{
-                    'tags': {
-                        'ceph.cluster_fsid': cephadm_module._cluster_fsid,
-                        'ceph.osd_fsid': 'uuid'
-                    },
-                    'type': 'data'
-                }, {
-                    'tags': {
-                        'ceph.cluster_fsid': cephadm_module._cluster_fsid,
-                        'ceph.osd_fsid': 'uuid'
-                    },
-                    'type': 'db'
-                }]
-            }
-            _run_cephadm.return_value = (json.dumps(ceph_volume_lvm_list), '', 0)
-            _run_cephadm.reset_mock()
+            def _ceph_volume_list(s, host, entity, cmd, **kwargs):
+                logging.info(f'ceph-volume cmd: {cmd}')
+                if 'raw' in cmd:
+                    return json.dumps({
+                        "21a4209b-f51b-4225-81dc-d2dca5b8b2f5": {
+                            "ceph_fsid": "64c84f19-fe1d-452a-a731-ab19dc144aa8",
+                            "device": "/dev/loop0",
+                            "osd_id": 21,
+                            "osd_uuid": "21a4209b-f51b-4225-81dc-d2dca5b8b2f5",
+                            "type": "bluestore"
+                        },
+                    }), '', 0
+                if 'lvm' in cmd:
+                    return json.dumps({
+                        '1': [{
+                            'tags': {
+                                'ceph.cluster_fsid': cephadm_module._cluster_fsid,
+                                'ceph.osd_fsid': 'uuid'
+                            },
+                            'type': 'data'
+                        }, {
+                            'tags': {
+                                'ceph.cluster_fsid': cephadm_module._cluster_fsid,
+                                'ceph.osd_fsid': 'uuid'
+                            },
+                            'type': 'db'
+                        }]
+                    }), '', 0
+                return '{}', '', 0
+            _run_cephadm.reset_mock(return_value=True)
+            _run_cephadm.side_effect = _ceph_volume_list
             assert cephadm_module._osd_activate(
                 ['test']).stdout == "Created osd(s) 1 on host 'test'"
             assert _run_cephadm.mock_calls == [
@@ -1512,4 +1547,6 @@ Traceback (most recent call last):
                           ['--name', 'osd.1', '--meta-json', mock.ANY,
                               '--config-json', '-', '--osd-fsid', 'uuid'],
                           stdin=mock.ANY, image=''),
+                mock.call('test', 'osd', 'ceph-volume',
+                          ['--', 'raw', 'list', '--format', 'json'], no_fsid=False, image=''),
             ]