From 28e33f3025d03008b46b56989e5d939682b5720d Mon Sep 17 00:00:00 2001 From: Jan Fajerski Date: Mon, 14 Sep 2020 11:32:38 +0200 Subject: [PATCH] python-common drive-groups: allow strings for wal and db size This enables user to pass sizes like "10G", which batch now understands. Signed-off-by: Jan Fajerski --- .../ceph/deployment/drive_group.py | 30 ++++++++----------- .../ceph/tests/test_disk_selector.py | 26 ++++++++++++---- .../ceph/tests/test_drive_group.py | 30 +++++++++++++++++-- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/python-common/ceph/deployment/drive_group.py b/src/python-common/ceph/deployment/drive_group.py index 21371d3c24c4f..86c229bed68de 100644 --- a/src/python-common/ceph/deployment/drive_group.py +++ b/src/python-common/ceph/deployment/drive_group.py @@ -4,7 +4,7 @@ from ceph.deployment.inventory import Device from ceph.deployment.service_spec import ServiceSpecValidationError, ServiceSpec, PlacementSpec try: - from typing import Optional, List, Dict, Any + from typing import Optional, List, Dict, Any, Union except ImportError: pass @@ -159,9 +159,9 @@ class DriveGroupSpec(ServiceSpec): db_slots=None, # type: Optional[int] wal_slots=None, # type: Optional[int] osd_id_claims=None, # type: Optional[Dict[str, List[str]]] - block_db_size=None, # type: Optional[int] - block_wal_size=None, # type: Optional[int] - journal_size=None, # type: Optional[int] + block_db_size=None, # type: Union[int, str, None] + block_wal_size=None, # type: Union[int, str, None] + journal_size=None, # type: Union[int, str, None] service_type=None, # type: Optional[str] unmanaged=False, # type: bool filter_logic='AND', # type: str @@ -186,13 +186,13 @@ class DriveGroupSpec(ServiceSpec): self.journal_devices = journal_devices #: Set (or override) the "bluestore_block_wal_size" value, in bytes - self.block_wal_size = block_wal_size + self.block_wal_size: Union[int, str, None] = block_wal_size #: Set (or override) the "bluestore_block_db_size" value, in bytes - self.block_db_size = block_db_size + self.block_db_size: Union[int, str, None] = block_db_size #: set journal_size in bytes - self.journal_size = journal_size + self.journal_size: Union[int, str, None] = journal_size #: Number of osd daemons per "DATA" device. #: To fully utilize nvme devices multiple osds are required. @@ -264,12 +264,6 @@ class DriveGroupSpec(ServiceSpec): raise DriveGroupValidationError( "Feature <{}> is not supported".format(applied_filter)) - for key in ('block_wal_size', 'block_db_size', 'journal_size'): - if key in json_drive_group: - if isinstance(json_drive_group[key], str): - from ceph.deployment.drive_selection import SizeMatcher - json_drive_group[key] = SizeMatcher.str_to_byte(json_drive_group[key]) - try: args = {k: (DeviceSelection.from_json(v) if k.endswith('_devices') else v) for k, v in json_drive_group.items()} @@ -303,10 +297,12 @@ class DriveGroupSpec(ServiceSpec): raise DriveGroupValidationError(f"{self.objectstore} is not supported. Must be " f"one of ('bluestore')") - if self.block_wal_size is not None and type(self.block_wal_size) != int: - raise DriveGroupValidationError('block_wal_size must be of type int') - if self.block_db_size is not None and type(self.block_db_size) != int: - raise DriveGroupValidationError('block_db_size must be of type int') + if self.block_wal_size is not None and type(self.block_wal_size) not in [int, str]: + raise DriveGroupValidationError('block_wal_size must be of type int or string') + if self.block_db_size is not None and type(self.block_db_size) not in [int, str]: + raise DriveGroupValidationError('block_db_size must be of type int or string') + if self.journal_size is not None and type(self.journal_size) not in [int, str]: + raise DriveGroupValidationError('journal_size must be of type int or string') if self.filter_logic not in ['AND', 'OR']: raise DriveGroupValidationError('filter_logic must be either or ') diff --git a/src/python-common/ceph/tests/test_disk_selector.py b/src/python-common/ceph/tests/test_disk_selector.py index 6931544d45313..579b03f0c5d49 100644 --- a/src/python-common/ceph/tests/test_disk_selector.py +++ b/src/python-common/ceph/tests/test_disk_selector.py @@ -3,7 +3,8 @@ import pytest from ceph.deployment.inventory import Devices, Device -from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection +from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection, \ + DriveGroupValidationError from ceph.deployment import drive_selection from ceph.deployment.service_spec import PlacementSpec @@ -277,6 +278,7 @@ class TestDriveGroup(object): disk_format='bluestore'): raw_sample_bluestore = { 'service_type': 'osd', + 'service_id': 'foo', 'placement': {'host_pattern': 'data*'}, 'data_devices': { 'size': '30G:50G', @@ -302,6 +304,7 @@ class TestDriveGroup(object): } raw_sample_filestore = { 'service_type': 'osd', + 'service_id': 'foo', 'placement': {'host_pattern': 'data*'}, 'objectstore': 'filestore', 'data_devices': { @@ -325,6 +328,7 @@ class TestDriveGroup(object): if empty: raw_sample = { 'service_type': 'osd', + 'service_id': 'foo', 'placement': {'host_pattern': 'data*'}, 'data_devices': { 'all': True @@ -332,6 +336,11 @@ class TestDriveGroup(object): } dgo = DriveGroupSpec.from_json(raw_sample) + if disk_format == 'filestore': + with pytest.raises(DriveGroupValidationError): + dgo.validate() + else: + dgo.validate() return dgo return make_sample_data @@ -362,7 +371,7 @@ class TestDriveGroup(object): def test_block_wal_size_prop(self, test_fix): test_fix = test_fix() - assert test_fix.block_wal_size == 5000000000 + assert test_fix.block_wal_size == '5G' def test_block_wal_size_prop_empty(self, test_fix): test_fix = test_fix(empty=True) @@ -370,7 +379,7 @@ class TestDriveGroup(object): def test_block_db_size_prop(self, test_fix): test_fix = test_fix() - assert test_fix.block_db_size == 10000000000 + assert test_fix.block_db_size == '10G' def test_block_db_size_prop_empty(self, test_fix): test_fix = test_fix(empty=True) @@ -431,7 +440,7 @@ class TestDriveGroup(object): def test_journal_size(self, test_fix): test_fix = test_fix(disk_format='filestore') - assert test_fix.journal_size == 5000000000 + assert test_fix.journal_size == '5G' def test_osds_per_device(self, test_fix): test_fix = test_fix(osds_per_device='3') @@ -489,13 +498,17 @@ class TestDriveSelection(object): testdata = [ ( - DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), data_devices=DeviceSelection(all=True)), + DriveGroupSpec( + placement=PlacementSpec(host_pattern='*'), + service_id='foobar', + data_devices=DeviceSelection(all=True)), _mk_inventory(_mk_device() * 5), ['/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde'], [] ), ( DriveGroupSpec( placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(all=True, limit=3), db_devices=DeviceSelection(all=True) ), @@ -505,6 +518,7 @@ class TestDriveSelection(object): ( DriveGroupSpec( placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False) ), @@ -514,6 +528,7 @@ class TestDriveSelection(object): ( DriveGroupSpec( placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False) ), @@ -523,6 +538,7 @@ class TestDriveSelection(object): ( DriveGroupSpec( placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False) ), diff --git a/src/python-common/ceph/tests/test_drive_group.py b/src/python-common/ceph/tests/test_drive_group.py index 85b2f3c217cd5..a98c178e7d373 100644 --- a/src/python-common/ceph/tests/test_drive_group.py +++ b/src/python-common/ceph/tests/test_drive_group.py @@ -63,13 +63,19 @@ def test_DriveGroup_fail(test_input): def test_drivegroup_pattern(): - dg = DriveGroupSpec(PlacementSpec(host_pattern='node[1-3]'), data_devices=DeviceSelection(all=True)) + dg = DriveGroupSpec( + PlacementSpec(host_pattern='node[1-3]'), + service_id='foobar', + data_devices=DeviceSelection(all=True)) assert dg.placement.filter_matching_hostspecs([HostSpec('node{}'.format(i)) for i in range(10)]) == ['node1', 'node2', 'node3'] def test_drive_selection(): devs = DeviceSelection(paths=['/dev/sda']) - spec = DriveGroupSpec(PlacementSpec('node_name'), data_devices=devs) + spec = DriveGroupSpec( + PlacementSpec('node_name'), + service_id='foobar', + data_devices=devs) assert all([isinstance(x, Device) for x in spec.data_devices.paths]) assert spec.data_devices.paths[0].path == '/dev/sda' @@ -79,8 +85,10 @@ def test_drive_selection(): def test_ceph_volume_command_0(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(all=True) ) + spec.validate() inventory = _mk_inventory(_mk_device()*2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, []).run() @@ -89,9 +97,11 @@ def test_ceph_volume_command_0(): def test_ceph_volume_command_1(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False) ) + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, []).run() @@ -101,10 +111,12 @@ def test_ceph_volume_command_1(): def test_ceph_volume_command_2(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(size='200GB:350GB', rotational=True), db_devices=DeviceSelection(size='200GB:350GB', rotational=False), wal_devices=DeviceSelection(size='10G') ) + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 + _mk_device(rotational=False, size="300.00 GB")*2 + _mk_device(size="10.0 GB", rotational=False)*2 @@ -118,11 +130,13 @@ def test_ceph_volume_command_2(): def test_ceph_volume_command_3(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(size='200GB:350GB', rotational=True), db_devices=DeviceSelection(size='200GB:350GB', rotational=False), wal_devices=DeviceSelection(size='10G'), encrypted=True ) + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 + _mk_device(rotational=False, size="300.00 GB")*2 + _mk_device(size="10.0 GB", rotational=False)*2 @@ -137,6 +151,7 @@ def test_ceph_volume_command_3(): def test_ceph_volume_command_4(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(size='200GB:350GB', rotational=True), db_devices=DeviceSelection(size='200GB:350GB', rotational=False), wal_devices=DeviceSelection(size='10G'), @@ -145,6 +160,7 @@ def test_ceph_volume_command_4(): osds_per_device=3, encrypted=True ) + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 + _mk_device(rotational=False, size="300.00 GB")*2 + _mk_device(size="10.0 GB", rotational=False)*2 @@ -159,9 +175,12 @@ def test_ceph_volume_command_4(): def test_ceph_volume_command_5(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(rotational=True), objectstore='filestore' ) + with pytest.raises(DriveGroupValidationError): + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True)*2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, []).run() @@ -170,11 +189,14 @@ def test_ceph_volume_command_5(): def test_ceph_volume_command_6(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(rotational=False), journal_devices=DeviceSelection(rotational=True), journal_size='500M', objectstore='filestore' ) + with pytest.raises(DriveGroupValidationError): + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, []).run() @@ -185,9 +207,11 @@ def test_ceph_volume_command_6(): def test_ceph_volume_command_7(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(all=True), osd_id_claims={'host1': ['0', '1']} ) + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True)*2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, ['0', '1']).run() @@ -196,11 +220,13 @@ def test_ceph_volume_command_7(): def test_ceph_volume_command_8(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), + service_id='foobar', data_devices=DeviceSelection(rotational=True, model='INTEL SSDS'), db_devices=DeviceSelection(model='INTEL SSDP'), filter_logic='OR', osd_id_claims={} ) + spec.validate() inventory = _mk_inventory(_mk_device(rotational=True, size='1.82 TB', model='ST2000DM001-1ER1') + # data _mk_device(rotational=False, size="223.0 GB", model='INTEL SSDSC2KG24') + # data _mk_device(rotational=False, size="349.0 GB", model='INTEL SSDPED1K375GA') # wal/db -- 2.39.5