From d94b72804870e06f884a873ba12c585e5c70f2dc Mon Sep 17 00:00:00 2001 From: Joshua Schmid Date: Wed, 1 Jul 2020 11:10:14 +0200 Subject: [PATCH] python-common/drivegroups: implement option to change filter_logic Filters are applied to disks using an AND gate by default. In some scenarios you can only achieve the desired behavior when using an OR gate without expanding the available filters. This patch allows to switch between `AND` and `OR` gates. Signed-off-by: Joshua Schmid (cherry picked from commit f00fba87925f6f72448128649694d98b4df3e748) --- src/pybind/mgr/cephadm/tests/test_spec.py | 1 + .../ceph/deployment/drive_group.py | 10 ++++++- .../deployment/drive_selection/selector.py | 18 +++++++++---- .../ceph/tests/test_drive_group.py | 26 +++++++++++++++++++ .../ceph/tests/test_service_spec.py | 1 + src/python-common/ceph/tests/utils.py | 8 +++--- 6 files changed, 55 insertions(+), 9 deletions(-) diff --git a/src/pybind/mgr/cephadm/tests/test_spec.py b/src/pybind/mgr/cephadm/tests/test_spec.py index 1a048d82d6f32..338759cf11ce7 100644 --- a/src/pybind/mgr/cephadm/tests/test_spec.py +++ b/src/pybind/mgr/cephadm/tests/test_spec.py @@ -102,6 +102,7 @@ def test_spec_octopus(spec_json): spec = j_c.pop('spec') j_c.update(spec) j_c.pop('objectstore', None) + j_c.pop('filter_logic', None) return j_c assert spec_json == convert_to_old_style_json(spec.to_json()) diff --git a/src/python-common/ceph/deployment/drive_group.py b/src/python-common/ceph/deployment/drive_group.py index 96f93fb8db62a..40d0cae7605fa 100644 --- a/src/python-common/ceph/deployment/drive_group.py +++ b/src/python-common/ceph/deployment/drive_group.py @@ -143,7 +143,7 @@ class DriveGroupSpec(ServiceSpec): "db_slots", "wal_slots", "block_db_size", "placement", "service_id", "service_type", "data_devices", "db_devices", "wal_devices", "journal_devices", "data_directories", "osds_per_device", "objectstore", "osd_id_claims", - "journal_size", "unmanaged" + "journal_size", "unmanaged", "filter_logic" ] def __init__(self, @@ -165,6 +165,7 @@ class DriveGroupSpec(ServiceSpec): journal_size=None, # type: Optional[int] service_type=None, # type: Optional[str] unmanaged=False, # type: bool + filter_logic='AND' # type: str ): assert service_type is None or service_type == 'osd' super(DriveGroupSpec, self).__init__('osd', service_id=service_id, @@ -215,6 +216,10 @@ class DriveGroupSpec(ServiceSpec): #: See :ref:`orchestrator-osd-replace` self.osd_id_claims = osd_id_claims or dict() + #: The logic gate we use to match disks with filters. + #: defaults to 'AND' + self.filter_logic = filter_logic.upper() + @classmethod def _from_json_impl(cls, json_drive_group): # type: (dict) -> DriveGroupSpec @@ -296,6 +301,9 @@ class DriveGroupSpec(ServiceSpec): if self.block_db_size is not None and type(self.block_db_size) != int: raise DriveGroupValidationError('block_db_size must be of type int') + if self.filter_logic not in ['AND', 'OR']: + raise DriveGroupValidationError('filter_logic must be either or ') + def __repr__(self): keys = [ key for key in self._supported_features if getattr(self, key) is not None diff --git a/src/python-common/ceph/deployment/drive_selection/selector.py b/src/python-common/ceph/deployment/drive_selection/selector.py index cdc21caa52ff2..921f125616c25 100644 --- a/src/python-common/ceph/deployment/drive_selection/selector.py +++ b/src/python-common/ceph/deployment/drive_selection/selector.py @@ -134,11 +134,19 @@ class DriveSelection(object): if disk in devices: continue - if not all(m.compare(disk) for m in FilterGenerator(device_filter)): - logger.debug( - "Ignoring disk {}. Filter did not match".format( - disk.path)) - continue + if self.spec.filter_logic == 'AND': + if not all(m.compare(disk) for m in FilterGenerator(device_filter)): + logger.debug( + "Ignoring disk {}. Not all filter did match the disk".format( + disk.path)) + continue + + if self.spec.filter_logic == 'OR': + if not any(m.compare(disk) for m in FilterGenerator(device_filter)): + logger.debug( + "Ignoring disk {}. No filter matched the disk".format( + disk.path)) + continue logger.debug('Adding disk {}'.format(disk.path)) devices.append(disk) diff --git a/src/python-common/ceph/tests/test_drive_group.py b/src/python-common/ceph/tests/test_drive_group.py index ba3cd6ec949f9..85b2f3c217cd5 100644 --- a/src/python-common/ceph/tests/test_drive_group.py +++ b/src/python-common/ceph/tests/test_drive_group.py @@ -43,6 +43,16 @@ placement: host_pattern: '*' data_devices: limit: 1 +"""), + + yaml.safe_load(""" +service_type: osd +service_id: mydg +placement: + host_pattern: '*' +data_devices: + all: True +filter_logic: XOR """) ) ]) @@ -182,3 +192,19 @@ def test_ceph_volume_command_7(): sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, ['0', '1']).run() assert cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --osd-ids 0 1 --yes --no-systemd' + + +def test_ceph_volume_command_8(): + spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + data_devices=DeviceSelection(rotational=True, model='INTEL SSDS'), + db_devices=DeviceSelection(model='INTEL SSDP'), + filter_logic='OR', + osd_id_claims={} + ) + inventory = _mk_inventory(_mk_device(rotational=True, size='1.82 TB', model='ST2000DM001-1ER1') + # data + _mk_device(rotational=False, size="223.0 GB", model='INTEL SSDSC2KG24') + # data + _mk_device(rotational=False, size="349.0 GB", model='INTEL SSDPED1K375GA') # wal/db + ) + sel = drive_selection.DriveSelection(spec, inventory) + cmd = translate.to_ceph_volume(sel, []).run() + assert cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --db-devices /dev/sdc --yes --no-systemd' diff --git a/src/python-common/ceph/tests/test_service_spec.py b/src/python-common/ceph/tests/test_service_spec.py index 7d12ff5d35820..5f15ae45a90e5 100644 --- a/src/python-common/ceph/tests/test_service_spec.py +++ b/src/python-common/ceph/tests/test_service_spec.py @@ -151,6 +151,7 @@ spec: model: MC-55-44-XZ db_devices: model: SSD-123-foo + filter_logic: AND objectstore: bluestore wal_devices: model: NVME-QQQQ-987 diff --git a/src/python-common/ceph/tests/utils.py b/src/python-common/ceph/tests/utils.py index ebba43094bc10..04b8a4e389554 100644 --- a/src/python-common/ceph/tests/utils.py +++ b/src/python-common/ceph/tests/utils.py @@ -8,12 +8,14 @@ except ImportError: def _mk_device(rotational=True, locked=False, - size="394.27 GB"): + size="394.27 GB", + vendor='Vendor', + model='Model'): return [Device( path='??', sys_api={ "rotational": '1' if rotational else '0', - "vendor": "Vendor", + "vendor": vendor, "human_readable_size": size, "partitions": {}, "locked": int(locked), @@ -21,7 +23,7 @@ def _mk_device(rotational=True, "removable": "0", "path": "??", "support_discard": "", - "model": "Model", + "model": model, "ro": "0", "nr_requests": "128", "size": 423347879936 # ignore coversion from human_readable_size -- 2.39.5