From 585992a1986e4ff76d80da42a8fad7c7dd1c492f Mon Sep 17 00:00:00 2001 From: Shraddha Agrawal Date: Thu, 29 Jan 2026 09:58:00 +0530 Subject: [PATCH] ceph-volume: support crimson osd binary Prior to this commit, ceph-volume was using hardcoded OSD binary to issue commands (eg - to perform mkfs, etc). This commit enables ceph-volume to start supporting crimson OSDs. A new argument, --osd-type is introduced with the default value classic. When this parameter is set to 'crimson', ceph-osd-crimson binary will be used to execute OSD commands. Signed-off-by: Shraddha Agrawal --- .../ceph_volume/devices/lvm/activate.py | 8 ++++++ .../ceph_volume/devices/lvm/batch.py | 9 ++++++ .../ceph_volume/devices/lvm/common.py | 7 +++++ .../ceph_volume/devices/raw/activate.py | 9 +++++- .../ceph_volume/devices/raw/common.py | 8 ++++++ .../objectstore/baseobjectstore.py | 11 ++++++-- .../tests/devices/lvm/test_activate.py | 12 +++++--- .../tests/objectstore/test_baseobjectstore.py | 1 + .../ceph_volume/tests/util/test_prepare.py | 6 ++-- src/pybind/mgr/cephadm/module.py | 5 ++-- src/pybind/mgr/cephadm/tests/test_cephadm.py | 14 ++++++---- .../ceph/deployment/translate.py | 4 +++ .../ceph/tests/test_drive_group.py | 28 +++++++++---------- 13 files changed, 91 insertions(+), 31 deletions(-) diff --git a/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/src/ceph-volume/ceph_volume/devices/lvm/activate.py index 60ef423ad9b..bd3fab84229 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -88,6 +88,14 @@ class Activate(object): action='store_true', help='Do not use a tmpfs mount for OSD data dir' ) + parser.add_argument( + '--osd-type', + dest='osd_type', + help='The Ceph OSD type to use.', + default='classic', + choices=['classic', 'crimson'], + type=str, + ) if self.argv is None: self.argv = [] if len(self.argv) == 0 and self.args is None: diff --git a/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/src/ceph-volume/ceph_volume/devices/lvm/batch.py index 9ed46d7afc3..7bf774dbbad 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/batch.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/batch.py @@ -290,6 +290,14 @@ class Batch(object): default=None, help="Additional cryptsetup luksOpen options (use the same syntax as the cryptsetup CLI)", ) + parser.add_argument( + '--osd-type', + dest='osd_type', + help='The Ceph OSD type to use.', + default='classic', + choices=['classic', 'crimson'], + type=str, + ) self.args = parser.parse_args(argv) if self.args.bluestore: self.args.objectstore = 'bluestore' @@ -386,6 +394,7 @@ class Batch(object): defaults = common.get_default_args() global_args = [ 'objectstore', + 'osd_type', 'bluestore', 'dmcrypt', 'with_tpm', diff --git a/src/ceph-volume/ceph_volume/devices/lvm/common.py b/src/ceph-volume/ceph_volume/devices/lvm/common.py index fa39ac6c962..d2a7310e615 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/common.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/common.py @@ -102,6 +102,13 @@ common_args: Dict[str, Any] = { 'action': 'store_true', 'help': 'Skip creating and enabling systemd units and starting OSD services when activating', }, + '--osd-type': { + 'dest': 'osd_type', + 'help': 'The Ceph OSD type to use.', + 'default': 'classic', + 'choices': ['classic', 'crimson'], + 'type': str, + }, } bluestore_args: Dict[str, Any] = { diff --git a/src/ceph-volume/ceph_volume/devices/raw/activate.py b/src/ceph-volume/ceph_volume/devices/raw/activate.py index 0cec810728b..3fecf06ec22 100644 --- a/src/ceph-volume/ceph_volume/devices/raw/activate.py +++ b/src/ceph-volume/ceph_volume/devices/raw/activate.py @@ -83,7 +83,14 @@ class Activate(object): action='store_true', help='Do not use a tmpfs mount for OSD data dir' ) - + parser.add_argument( + '--osd-type', + dest='osd_type', + help='The Ceph OSD type to use.', + default='classic', + choices=['classic', 'crimson'], + type=str, + ) if not self.argv: print(sub_command_help) return diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py index 77db2f7daf9..27a2b67a9c0 100644 --- a/src/ceph-volume/ceph_volume/devices/raw/common.py +++ b/src/ceph-volume/ceph_volume/devices/raw/common.py @@ -70,4 +70,12 @@ def create_parser(prog: str, description: str) -> argparse.ArgumentParser: default=None, type=arg_validators.valid_osd_id, ) + parser.add_argument( + '--osd-type', + dest='osd_type', + help='The Ceph OSD type to use.', + default='classic', + choices=['classic', 'crimson'], + type=str, + ) return parser diff --git a/src/ceph-volume/ceph_volume/objectstore/baseobjectstore.py b/src/ceph-volume/ceph_volume/objectstore/baseobjectstore.py index 863819e736a..e3844efc282 100644 --- a/src/ceph-volume/ceph_volume/objectstore/baseobjectstore.py +++ b/src/ceph-volume/ceph_volume/objectstore/baseobjectstore.py @@ -34,6 +34,7 @@ class BaseObjectStore: self.cephx_lockbox_secret: str = '' self.objectstore: str = getattr(args, "objectstore", '') self.osd_mkfs_cmd: List[str] = [] + self.osd_type: str = getattr(args, "osd_type", '') self.block_device_path: str = '' self.dmcrypt_key: str = encryption_utils.create_dmcrypt_key() self.with_tpm: int = int(getattr(self.args, 'with_tpm', False)) @@ -139,7 +140,13 @@ class BaseObjectStore: def get_osd_path(self) -> str: return '/var/lib/ceph/osd/%s-%s/' % (conf.cluster, self.osd_id) + def get_default_entrypoint_cmd(self) -> str: + if self.osd_type == "crimson": + return "ceph-osd-crimson" + return "ceph-osd" + def build_osd_mkfs_cmd(self) -> List[str]: + base_mkfs_cmd = self.get_default_entrypoint_cmd() self.supplementary_command = [ '--osd-data', self.osd_path, '--osd-uuid', self.osd_fsid, @@ -147,7 +154,7 @@ class BaseObjectStore: '--setgroup', 'ceph' ] self.osd_mkfs_cmd = [ - 'ceph-osd', + base_mkfs_cmd, '--cluster', conf.cluster, '--osd-objectstore', self.objectstore, '--mkfs', @@ -257,4 +264,4 @@ class BaseObjectStore: terminal_verbose=True, show_command=True) if rc: - raise RuntimeError(f"Can't add BlueStore label '{key}' to device {device}: {err}") \ No newline at end of file + raise RuntimeError(f"Can't add BlueStore label '{key}' to device {device}: {err}") diff --git a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py index ba30ff4cd61..6254bc38a85 100644 --- a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py +++ b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py @@ -225,7 +225,8 @@ class TestActivateAll(object): no_tmpfs=False, objectstore='bluestore', osd_fsid=None, - osd_id=None), + osd_id=None, + osd_type='classic'), osd_id='0', osd_fsid='957d22b7-24ce-466a-9883-b8218eaa1634'), call(Namespace(activate_all=True, @@ -235,7 +236,8 @@ class TestActivateAll(object): no_tmpfs=False, objectstore='bluestore', osd_fsid=None, - osd_id=None), + osd_id=None, + osd_type='classic'), osd_id='1', osd_fsid='d0f3e4ad-e52a-4520-afc0-a8789a96ce8b') ] @@ -255,7 +257,8 @@ class TestActivateAll(object): no_tmpfs=False, objectstore='bluestore', osd_fsid=None, - osd_id=None), + osd_id=None, + osd_type='classic'), osd_id='0', osd_fsid='957d22b7-24ce-466a-9883-b8218eaa1634'), call(Namespace(activate_all=True, @@ -265,7 +268,8 @@ class TestActivateAll(object): no_tmpfs=False, objectstore='bluestore', osd_fsid=None, - osd_id=None), + osd_id=None, + osd_type='classic'), osd_id='1', osd_fsid='d0f3e4ad-e52a-4520-afc0-a8789a96ce8b') ] diff --git a/src/ceph-volume/ceph_volume/tests/objectstore/test_baseobjectstore.py b/src/ceph-volume/ceph_volume/tests/objectstore/test_baseobjectstore.py index 3233fafdbac..fba3bbc48f8 100644 --- a/src/ceph-volume/ceph_volume/tests/objectstore/test_baseobjectstore.py +++ b/src/ceph-volume/ceph_volume/tests/objectstore/test_baseobjectstore.py @@ -96,6 +96,7 @@ class TestBaseObjectStore: bo.objectstore = 'my-fake-objectstore' bo.osd_id = '123' bo.monmap = '/etc/ceph/ceph.monmap' + bo.osd_type = 'classic' result = bo.build_osd_mkfs_cmd() assert result == ['ceph-osd', diff --git a/src/ceph-volume/ceph_volume/tests/util/test_prepare.py b/src/ceph-volume/ceph_volume/tests/util/test_prepare.py index 414e2f4d1e1..f87f31c1c64 100644 --- a/src/ceph-volume/ceph_volume/tests/util/test_prepare.py +++ b/src/ceph-volume/ceph_volume/tests/util/test_prepare.py @@ -296,7 +296,8 @@ class TestMkfs(object): stub_call(([], [], 1)) o = objectstore(osd_id='1', osd_fsid='asdf-1234', - cephx_secret='keyring') + cephx_secret='keyring', + osd_type='classic') with pytest.raises(RuntimeError) as error: o.osd_mkfs() assert "Command failed with exit code 1" in str(error.value) @@ -308,7 +309,8 @@ class TestMkfs(object): o = objectstore(osd_id='1', objectstore='bluestore', osd_fsid='asdf-1234', - cephx_secret='keyring') + cephx_secret='keyring', + osd_type='classic') with pytest.raises(RuntimeError) as error: o.osd_mkfs() expected = ' '.join([ diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 91789c8673b..ddfefc1f329 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -33,7 +33,7 @@ from prettytable import PrettyTable from ceph.cephadm.images import DefaultImages from ceph.deployment import inventory -from ceph.deployment.drive_group import DriveGroupSpec +from ceph.deployment.drive_group import DriveGroupSpec, OSDType from ceph.deployment.service_spec import ( ServiceSpec, PlacementSpec, @@ -3056,7 +3056,8 @@ Then run the following: data_devices=DeviceSelection(paths=devices), unmanaged=False, method=drive_group.method, - objectstore=drive_group.objectstore + objectstore=drive_group.objectstore, + osd_type=OSDType(drive_group.osd_type) ) self.log.info(f"Creating OSDs with service ID: {drive_group.service_id} on {host}:{device_list}") diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index 50119e97e02..a9d40ec7f07 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -1132,7 +1132,8 @@ class TestCephadm(object): _run_cephadm.assert_any_call( 'test', 'osd', 'ceph-volume', ['--config-json', '-', '--', 'lvm', 'batch', - '--no-auto', '/dev/sdb', '--objectstore', 'bluestore', '--yes', '--no-systemd'], + '--no-auto', '/dev/sdb', '--objectstore', 'bluestore', '--osd-type', 'classic', + '--yes', '--no-systemd'], env_vars=['CEPH_VOLUME_OSDSPEC_AFFINITY=foo'], error_ok=True, stdin='{"config": "", "keyring": ""}') _run_cephadm.assert_any_call( @@ -1178,7 +1179,8 @@ class TestCephadm(object): 'test', 'osd', 'ceph-volume', ['--config-json', '-', '--', 'lvm', 'batch', '--no-auto', '/dev/sdb', '--db-devices', '/dev/sdc', - '--wal-devices', '/dev/sdd', '--objectstore', 'bluestore', '--yes', '--no-systemd'], + '--wal-devices', '/dev/sdd', '--objectstore', 'bluestore', '--osd-type', 'classic', + '--yes', '--no-systemd'], env_vars=['CEPH_VOLUME_OSDSPEC_AFFINITY=noncollocated'], error_ok=True, stdin='{"config": "", "keyring": ""}', ) @@ -1266,15 +1268,15 @@ class TestCephadm(object): "devices, preview, exp_commands", [ # no preview and only one disk, prepare is used due the hack that is in place. - (['/dev/sda'], False, ["lvm batch --no-auto /dev/sda --objectstore bluestore --yes --no-systemd"]), + (['/dev/sda'], False, ["lvm batch --no-auto /dev/sda --objectstore bluestore --osd-type classic --yes --no-systemd"]), # no preview and multiple disks, uses batch (['/dev/sda', '/dev/sdb'], False, - ["CEPH_VOLUME_OSDSPEC_AFFINITY=test.spec lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --yes --no-systemd"]), + ["CEPH_VOLUME_OSDSPEC_AFFINITY=test.spec lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --osd-type classic --yes --no-systemd"]), # preview and only one disk needs to use batch again to generate the preview - (['/dev/sda'], True, ["lvm batch --no-auto /dev/sda --objectstore bluestore --yes --no-systemd --report --format json"]), + (['/dev/sda'], True, ["lvm batch --no-auto /dev/sda --objectstore bluestore --osd-type classic --yes --no-systemd --report --format json"]), # preview and multiple disks work the same (['/dev/sda', '/dev/sdb'], True, - ["CEPH_VOLUME_OSDSPEC_AFFINITY=test.spec lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --yes --no-systemd --report --format json"]), + ["CEPH_VOLUME_OSDSPEC_AFFINITY=test.spec lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --osd-type classic --yes --no-systemd --report --format json"]), ] ) @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) diff --git a/src/python-common/ceph/deployment/translate.py b/src/python-common/ceph/deployment/translate.py index c6bb654867b..fab0518e1a9 100644 --- a/src/python-common/ceph/deployment/translate.py +++ b/src/python-common/ceph/deployment/translate.py @@ -134,6 +134,10 @@ class to_ceph_volume(object): cmd += " --crush-device-class {}".format(d) cmd += " --objectstore {}".format(self.spec.objectstore) + + if self.spec.osd_type: + cmd += " --osd-type {}".format(self.spec.osd_type) + cmds.append(cmd) for i in range(len(cmds)): diff --git a/src/python-common/ceph/tests/test_drive_group.py b/src/python-common/ceph/tests/test_drive_group.py index 5252fcb689b..afcc5b1fec1 100644 --- a/src/python-common/ceph/tests/test_drive_group.py +++ b/src/python-common/ceph/tests/test_drive_group.py @@ -163,7 +163,7 @@ def test_ceph_volume_command_0(): inventory = _mk_inventory(_mk_device()*2) sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() - assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' + assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --osd-type classic --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' def test_ceph_volume_command_1(): @@ -177,7 +177,7 @@ def test_ceph_volume_command_1(): sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb ' - '--db-devices /dev/sdc /dev/sdd --objectstore bluestore --yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}' + '--db-devices /dev/sdc /dev/sdd --objectstore bluestore --osd-type classic --yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}' def test_ceph_volume_command_2(): @@ -196,7 +196,7 @@ def test_ceph_volume_command_2(): cmds = translate.to_ceph_volume(sel, []).run() assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb ' '--db-devices /dev/sdc /dev/sdd --wal-devices /dev/sde /dev/sdf --objectstore bluestore ' - '--yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}' + '--osd-type classic --yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}' def test_ceph_volume_command_3(): @@ -216,7 +216,7 @@ def test_ceph_volume_command_3(): cmds = translate.to_ceph_volume(sel, []).run() assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb ' '--db-devices /dev/sdc /dev/sdd ' - '--wal-devices /dev/sde /dev/sdf --objectstore bluestore --dmcrypt ' + '--wal-devices /dev/sde /dev/sdf --objectstore bluestore --osd-type classic --dmcrypt ' '--yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}' @@ -240,7 +240,7 @@ def test_ceph_volume_command_4(): cmds = translate.to_ceph_volume(sel, []).run() assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb ' '--db-devices /dev/sdc /dev/sdd --wal-devices /dev/sde /dev/sdf ' - '--block-wal-size 500M --block-db-size 500M --objectstore bluestore --dmcrypt ' + '--block-wal-size 500M --block-db-size 500M --objectstore bluestore --osd-type classic --dmcrypt ' '--osds-per-device 3 --yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}' @@ -286,7 +286,7 @@ def test_ceph_volume_command_7(): inventory = _mk_inventory(_mk_device(rotational=True)*2) sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, ['0', '1']).run() - assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --osd-ids 0 1 --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' + assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --osd-type classic --osd-ids 0 1 --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' def test_ceph_volume_command_8(): @@ -304,7 +304,7 @@ def test_ceph_volume_command_8(): ) sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() - assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --db-devices /dev/sdc --objectstore bluestore --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' + assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --db-devices /dev/sdc --objectstore bluestore --osd-type classic --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' def test_ceph_volume_command_9(): @@ -317,7 +317,7 @@ def test_ceph_volume_command_9(): inventory = _mk_inventory(_mk_device()*2) sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() - assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --data-allocate-fraction 0.8 --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' + assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --objectstore bluestore --osd-type classic --data-allocate-fraction 0.8 --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' @pytest.mark.parametrize("test_input_base", @@ -340,7 +340,7 @@ def test_ceph_volume_command_10(test_input_base): drive = drive_selection.DriveSelection(spec, spec.data_devices.paths) cmds = translate.to_ceph_volume(drive, []).run() - assert all(cmd == 'lvm batch --no-auto /dev/sda --crush-device-class ssd --objectstore bluestore --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' + assert all(cmd == 'lvm batch --no-auto /dev/sda --crush-device-class ssd --objectstore bluestore --osd-type classic --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' @pytest.mark.parametrize("test_input1", @@ -365,7 +365,7 @@ def test_ceph_volume_command_11(test_input1): spec.validate() drive = drive_selection.DriveSelection(spec, spec.data_devices.paths) cmds = translate.to_ceph_volume(drive, []).run() - assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --crush-device-class hdd --objectstore bluestore --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' + assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --crush-device-class hdd --objectstore bluestore --osd-type classic --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}' @pytest.mark.parametrize("test_input2", @@ -392,8 +392,8 @@ def test_ceph_volume_command_12(test_input2): cmds = translate.to_ceph_volume(drive, []).run() expected_cmds = [ - 'lvm batch --no-auto /dev/sdb --crush-device-class ssd --objectstore bluestore --yes --no-systemd', - 'lvm batch --no-auto /dev/sda --crush-device-class hdd --objectstore bluestore --yes --no-systemd', + 'lvm batch --no-auto /dev/sdb --crush-device-class ssd --objectstore bluestore --osd-type classic --yes --no-systemd', + 'lvm batch --no-auto /dev/sda --crush-device-class hdd --objectstore bluestore --osd-type classic --yes --no-systemd', ] assert len(cmds) == len(expected_cmds), f"Expected {expected_cmds} got {cmds}" assert all(cmd in cmds for cmd in expected_cmds), f'Expected {expected_cmds} got {cmds}' @@ -422,8 +422,8 @@ def test_ceph_volume_command_13(test_input3): cmds = translate.to_ceph_volume(drive, []).run() expected_cmds = [ - 'lvm batch --no-auto /dev/sdb --objectstore bluestore --yes --no-systemd', - 'lvm batch --no-auto /dev/sda --crush-device-class hdd --objectstore bluestore --yes --no-systemd', + 'lvm batch --no-auto /dev/sdb --objectstore bluestore --osd-type classic --yes --no-systemd', + 'lvm batch --no-auto /dev/sda --crush-device-class hdd --objectstore bluestore --osd-type classic --yes --no-systemd', ] assert len(cmds) == len(expected_cmds), f"Expected {expected_cmds} got {cmds}" assert all(cmd in cmds for cmd in expected_cmds), f'Expected {expected_cmds} got {cmds}' -- 2.47.3