python-common drive-groups: allow strings for wal and db size

This enables user to pass sizes like "10G", which batch now understands.

Signed-off-by: Jan Fajerski <jfajerski@suse.com>

diff --git a/src/python-common/ceph/deployment/drive_group.py b/src/python-common/ceph/deployment/drive_group.py
index 21371d3c24c4f7ad540cf98740ff8ce9641a6784..86c229bed68de09e2a2940e8aefa8e7f5543c545 100644 (file)
--- a/src/python-common/ceph/deployment/drive_group.py
+++ b/src/python-common/ceph/deployment/drive_group.py
@@ -4,7 +4,7 @@ from ceph.deployment.inventory import Device
 from ceph.deployment.service_spec import ServiceSpecValidationError, ServiceSpec, PlacementSpec
 
 try:
-    from typing import Optional, List, Dict, Any
+    from typing import Optional, List, Dict, Any, Union
 except ImportError:
     pass
 
@@ -159,9 +159,9 @@ class DriveGroupSpec(ServiceSpec):
                  db_slots=None,  # type: Optional[int]
                  wal_slots=None,  # type: Optional[int]
                  osd_id_claims=None,  # type: Optional[Dict[str, List[str]]]
-                 block_db_size=None,  # type: Optional[int]
-                 block_wal_size=None,  # type: Optional[int]
-                 journal_size=None,  # type: Optional[int]
+                 block_db_size=None,  # type: Union[int, str, None]
+                 block_wal_size=None,  # type: Union[int, str, None]
+                 journal_size=None,  # type: Union[int, str, None]
                  service_type=None,  # type: Optional[str]
                  unmanaged=False,  # type: bool
                  filter_logic='AND',  # type: str
@@ -186,13 +186,13 @@ class DriveGroupSpec(ServiceSpec):
         self.journal_devices = journal_devices
 
         #: Set (or override) the "bluestore_block_wal_size" value, in bytes
-        self.block_wal_size = block_wal_size
+        self.block_wal_size: Union[int, str, None] = block_wal_size
 
         #: Set (or override) the "bluestore_block_db_size" value, in bytes
-        self.block_db_size = block_db_size
+        self.block_db_size: Union[int, str, None] = block_db_size
 
         #: set journal_size in bytes
-        self.journal_size = journal_size
+        self.journal_size: Union[int, str, None] = journal_size
 
         #: Number of osd daemons per "DATA" device.
         #: To fully utilize nvme devices multiple osds are required.
@@ -264,12 +264,6 @@ class DriveGroupSpec(ServiceSpec):
                 raise DriveGroupValidationError(
                     "Feature <{}> is not supported".format(applied_filter))
 
-        for key in ('block_wal_size', 'block_db_size', 'journal_size'):
-            if key in json_drive_group:
-                if isinstance(json_drive_group[key], str):
-                    from ceph.deployment.drive_selection import SizeMatcher
-                    json_drive_group[key] = SizeMatcher.str_to_byte(json_drive_group[key])
-
         try:
             args = {k: (DeviceSelection.from_json(v) if k.endswith('_devices') else v) for k, v in
                     json_drive_group.items()}
@@ -303,10 +297,12 @@ class DriveGroupSpec(ServiceSpec):
             raise DriveGroupValidationError(f"{self.objectstore} is not supported. Must be "
                                             f"one of ('bluestore')")
 
-        if self.block_wal_size is not None and type(self.block_wal_size) != int:
-            raise DriveGroupValidationError('block_wal_size must be of type int')
-        if self.block_db_size is not None and type(self.block_db_size) != int:
-            raise DriveGroupValidationError('block_db_size must be of type int')
+        if self.block_wal_size is not None and type(self.block_wal_size) not in [int, str]:
+            raise DriveGroupValidationError('block_wal_size must be of type int or string')
+        if self.block_db_size is not None and type(self.block_db_size) not in [int, str]:
+            raise DriveGroupValidationError('block_db_size must be of type int or string')
+        if self.journal_size is not None and type(self.journal_size) not in [int, str]:
+            raise DriveGroupValidationError('journal_size must be of type int or string')
 
         if self.filter_logic not in ['AND', 'OR']:
             raise DriveGroupValidationError('filter_logic must be either <AND> or <OR>')

diff --git a/src/python-common/ceph/tests/test_disk_selector.py b/src/python-common/ceph/tests/test_disk_selector.py
index 6931544d45313071b72a018cbd6e305bb20ae541..579b03f0c5d4974de4621e262cd7e6dfe9c91c11 100644 (file)
--- a/src/python-common/ceph/tests/test_disk_selector.py
+++ b/src/python-common/ceph/tests/test_disk_selector.py
@@ -3,7 +3,8 @@ import pytest
 
 from ceph.deployment.inventory import Devices, Device
 
-from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection
+from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection, \
+        DriveGroupValidationError
 
 from ceph.deployment import drive_selection
 from ceph.deployment.service_spec import PlacementSpec
@@ -277,6 +278,7 @@ class TestDriveGroup(object):
                              disk_format='bluestore'):
             raw_sample_bluestore = {
                 'service_type': 'osd',
+                'service_id': 'foo',
                 'placement': {'host_pattern': 'data*'},
                 'data_devices': {
                     'size': '30G:50G',
@@ -302,6 +304,7 @@ class TestDriveGroup(object):
             }
             raw_sample_filestore = {
                 'service_type': 'osd',
+                'service_id': 'foo',
                 'placement': {'host_pattern': 'data*'},
                 'objectstore': 'filestore',
                 'data_devices': {
@@ -325,6 +328,7 @@ class TestDriveGroup(object):
             if empty:
                 raw_sample = {
                     'service_type': 'osd',
+                    'service_id': 'foo',
                     'placement': {'host_pattern': 'data*'},
                     'data_devices': {
                         'all': True
@@ -332,6 +336,11 @@ class TestDriveGroup(object):
                 }
 
             dgo = DriveGroupSpec.from_json(raw_sample)
+            if disk_format == 'filestore':
+                with pytest.raises(DriveGroupValidationError):
+                    dgo.validate()
+            else:
+                dgo.validate()
             return dgo
 
         return make_sample_data
@@ -362,7 +371,7 @@ class TestDriveGroup(object):
 
     def test_block_wal_size_prop(self, test_fix):
         test_fix = test_fix()
-        assert test_fix.block_wal_size == 5000000000
+        assert test_fix.block_wal_size == '5G'
 
     def test_block_wal_size_prop_empty(self, test_fix):
         test_fix = test_fix(empty=True)
@@ -370,7 +379,7 @@ class TestDriveGroup(object):
 
     def test_block_db_size_prop(self, test_fix):
         test_fix = test_fix()
-        assert test_fix.block_db_size == 10000000000
+        assert test_fix.block_db_size == '10G'
 
     def test_block_db_size_prop_empty(self, test_fix):
         test_fix = test_fix(empty=True)
@@ -431,7 +440,7 @@ class TestDriveGroup(object):
 
     def test_journal_size(self, test_fix):
         test_fix = test_fix(disk_format='filestore')
-        assert test_fix.journal_size == 5000000000
+        assert test_fix.journal_size == '5G'
 
     def test_osds_per_device(self, test_fix):
         test_fix = test_fix(osds_per_device='3')
@@ -489,13 +498,17 @@ class TestDriveSelection(object):
 
     testdata = [
         (
-            DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), data_devices=DeviceSelection(all=True)),
+            DriveGroupSpec(
+                placement=PlacementSpec(host_pattern='*'),
+                service_id='foobar',
+                data_devices=DeviceSelection(all=True)),
             _mk_inventory(_mk_device() * 5),
             ['/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde'], []
         ),
         (
             DriveGroupSpec(
                 placement=PlacementSpec(host_pattern='*'),
+                service_id='foobar',
                 data_devices=DeviceSelection(all=True, limit=3),
                 db_devices=DeviceSelection(all=True)
             ),
@@ -505,6 +518,7 @@ class TestDriveSelection(object):
         (
             DriveGroupSpec(
                 placement=PlacementSpec(host_pattern='*'),
+                service_id='foobar',
                 data_devices=DeviceSelection(rotational=True),
                 db_devices=DeviceSelection(rotational=False)
             ),
@@ -514,6 +528,7 @@ class TestDriveSelection(object):
         (
             DriveGroupSpec(
                 placement=PlacementSpec(host_pattern='*'),
+                service_id='foobar',
                 data_devices=DeviceSelection(rotational=True),
                 db_devices=DeviceSelection(rotational=False)
             ),
@@ -523,6 +538,7 @@ class TestDriveSelection(object):
         (
             DriveGroupSpec(
                 placement=PlacementSpec(host_pattern='*'),
+                service_id='foobar',
                 data_devices=DeviceSelection(rotational=True),
                 db_devices=DeviceSelection(rotational=False)
             ),

diff --git a/src/python-common/ceph/tests/test_drive_group.py b/src/python-common/ceph/tests/test_drive_group.py
index 85b2f3c217cd55ca9c8a3e8b5edf53a2e4bfdfb6..a98c178e7d37335ec5e1d517c41df2ad86201286 100644 (file)
--- a/src/python-common/ceph/tests/test_drive_group.py
+++ b/src/python-common/ceph/tests/test_drive_group.py
@@ -63,13 +63,19 @@ def test_DriveGroup_fail(test_input):
 
 
 def test_drivegroup_pattern():
-    dg = DriveGroupSpec(PlacementSpec(host_pattern='node[1-3]'), data_devices=DeviceSelection(all=True))
+    dg = DriveGroupSpec(
+        PlacementSpec(host_pattern='node[1-3]'),
+        service_id='foobar',
+        data_devices=DeviceSelection(all=True))
     assert dg.placement.filter_matching_hostspecs([HostSpec('node{}'.format(i)) for i in range(10)]) == ['node1', 'node2', 'node3']
 
 
 def test_drive_selection():
     devs = DeviceSelection(paths=['/dev/sda'])
-    spec = DriveGroupSpec(PlacementSpec('node_name'), data_devices=devs)
+    spec = DriveGroupSpec(
+        PlacementSpec('node_name'),
+        service_id='foobar',
+        data_devices=devs)
     assert all([isinstance(x, Device) for x in spec.data_devices.paths])
     assert spec.data_devices.paths[0].path == '/dev/sda'
 
@@ -79,8 +85,10 @@ def test_drive_selection():
 
 def test_ceph_volume_command_0():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(all=True)
                           )
+    spec.validate()
     inventory = _mk_inventory(_mk_device()*2)
     sel = drive_selection.DriveSelection(spec, inventory)
     cmd = translate.to_ceph_volume(sel, []).run()
@@ -89,9 +97,11 @@ def test_ceph_volume_command_0():
 
 def test_ceph_volume_command_1():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(rotational=True),
                           db_devices=DeviceSelection(rotational=False)
                           )
+    spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2)
     sel = drive_selection.DriveSelection(spec, inventory)
     cmd = translate.to_ceph_volume(sel, []).run()
@@ -101,10 +111,12 @@ def test_ceph_volume_command_1():
 
 def test_ceph_volume_command_2():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(size='200GB:350GB', rotational=True),
                           db_devices=DeviceSelection(size='200GB:350GB', rotational=False),
                           wal_devices=DeviceSelection(size='10G')
                           )
+    spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 +
                               _mk_device(rotational=False, size="300.00 GB")*2 +
                               _mk_device(size="10.0 GB", rotational=False)*2
@@ -118,11 +130,13 @@ def test_ceph_volume_command_2():
 
 def test_ceph_volume_command_3():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(size='200GB:350GB', rotational=True),
                           db_devices=DeviceSelection(size='200GB:350GB', rotational=False),
                           wal_devices=DeviceSelection(size='10G'),
                           encrypted=True
                           )
+    spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 +
                               _mk_device(rotational=False, size="300.00 GB")*2 +
                               _mk_device(size="10.0 GB", rotational=False)*2
@@ -137,6 +151,7 @@ def test_ceph_volume_command_3():
 
 def test_ceph_volume_command_4():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(size='200GB:350GB', rotational=True),
                           db_devices=DeviceSelection(size='200GB:350GB', rotational=False),
                           wal_devices=DeviceSelection(size='10G'),
@@ -145,6 +160,7 @@ def test_ceph_volume_command_4():
                           osds_per_device=3,
                           encrypted=True
                           )
+    spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 +
                               _mk_device(rotational=False, size="300.00 GB")*2 +
                               _mk_device(size="10.0 GB", rotational=False)*2
@@ -159,9 +175,12 @@ def test_ceph_volume_command_4():
 
 def test_ceph_volume_command_5():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(rotational=True),
                           objectstore='filestore'
                           )
+    with pytest.raises(DriveGroupValidationError):
+        spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True)*2)
     sel = drive_selection.DriveSelection(spec, inventory)
     cmd = translate.to_ceph_volume(sel, []).run()
@@ -170,11 +189,14 @@ def test_ceph_volume_command_5():
 
 def test_ceph_volume_command_6():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(rotational=False),
                           journal_devices=DeviceSelection(rotational=True),
                           journal_size='500M',
                           objectstore='filestore'
                           )
+    with pytest.raises(DriveGroupValidationError):
+        spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2)
     sel = drive_selection.DriveSelection(spec, inventory)
     cmd = translate.to_ceph_volume(sel, []).run()
@@ -185,9 +207,11 @@ def test_ceph_volume_command_6():
 
 def test_ceph_volume_command_7():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(all=True),
                           osd_id_claims={'host1': ['0', '1']}
                           )
+    spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True)*2)
     sel = drive_selection.DriveSelection(spec, inventory)
     cmd = translate.to_ceph_volume(sel, ['0', '1']).run()
@@ -196,11 +220,13 @@ def test_ceph_volume_command_7():
 
 def test_ceph_volume_command_8():
     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
+                          service_id='foobar',
                           data_devices=DeviceSelection(rotational=True, model='INTEL SSDS'),
                           db_devices=DeviceSelection(model='INTEL SSDP'),
                           filter_logic='OR',
                           osd_id_claims={}
                           )
+    spec.validate()
     inventory = _mk_inventory(_mk_device(rotational=True,  size='1.82 TB',  model='ST2000DM001-1ER1') +  # data
                               _mk_device(rotational=False, size="223.0 GB", model='INTEL SSDSC2KG24') +  # data
                               _mk_device(rotational=False, size="349.0 GB", model='INTEL SSDPED1K375GA')  # wal/db