python-common: improve host matching funcs

Signed-off-by: Joshua Schmid <jschmid@suse.de>
(cherry picked from commit d2aa0543d924d659fa0c5321b3fa9c607a44b5e3)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 295eaae46b069d68c42971d585c26757dd39f5f1..db497013d51400499fbe58a7bd77c935eed0679e 100644 (file)
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1526,14 +1526,13 @@ you may want to run:
         if not osdspecs:
             self.log.debug("No OSDSpecs found")
             return []
-        # TODO: adapt this when we change patter_matches_hosts with https://github.com/ceph/ceph/pull/34860
-        return sum([spec.placement.pattern_matches_hosts(self.cache.get_hosts()) for spec in osdspecs], [])
+        return sum([spec.placement.filter_matching_hosts(self._get_hosts) for spec in osdspecs], [])
 
     def resolve_osdspecs_for_host(self, host):
         matching_specs = []
         self.log.debug(f"Finding OSDSpecs for host: <{host}>")
         for spec in self.spec_store.find('osd'):
-            if host in spec.placement.pattern_matches_hosts(self.cache.get_hosts()):
+            if host in spec.placement.filter_matching_hosts(self._get_hosts):
                 self.log.debug(f"Found OSDSpecs for host: <{host}> -> <{spec}>")
                 matching_specs.append(spec)
         return matching_specs

diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
index 44228731ba0085646798b0858ad2312a9cf6e1a1..4b12d8e781a1834688d57a6ece0205de7733ad35 100644 (file)
--- a/src/pybind/mgr/rook/module.py
+++ b/src/pybind/mgr/rook/module.py
@@ -460,7 +460,6 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
         return self._service_add_decorate('RGW', spec,
                                           self.rook_cluster.apply_objectstore)
 
-
     def apply_nfs(self, spec):
         # type: (NFSServiceSpec) -> RookCompletion
         num = spec.placement.count
@@ -478,7 +477,7 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
         )
 
     def create_osds(self, drive_group):
-        # type: (DriveGroupSpec) -> orchestrator.Completion
+        # type: (DriveGroupSpec) -> RookCompletion
         """ Creates OSDs from a drive group specification.
 
         $: ceph orch osd create -i <dg.file>
@@ -492,9 +491,10 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
         if drive_group.data_directories:
             targets += drive_group.data_directories
 
-        def execute():
-            # type: () -> orchestrator.Completion
-            matching_hosts = drive_group.placement.filter_matching_hosts(self.get_hosts)
+        def execute(all_hosts_):
+            # type: (List[orchestrator.HostSpec]) -> orchestrator.Completion
+            all_hosts = [h.hostname for h in all_hosts_]
+            matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts)
 
             assert len(matching_hosts) == 1
 
@@ -549,7 +549,7 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
 
             return found is not None
 
-        c = execute()
+        c = self.get_hosts().then(execute)
         return c
 
     def blink_device_light(self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> RookCompletion:

diff --git a/src/pybind/mgr/test_orchestrator/module.py b/src/pybind/mgr/test_orchestrator/module.py
index bf807437708acd0e2e05f7ff081288c4d83de8f1..d3803db7e44e49794c4568b7b90777dae7c0e408 100644 (file)
--- a/src/pybind/mgr/test_orchestrator/module.py
+++ b/src/pybind/mgr/test_orchestrator/module.py
@@ -260,9 +260,10 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator):
         def run(all_hosts):
             # type: (List[orchestrator.HostSpec]) -> None
             drive_group.validate()
-            if drive_group.placement.host_pattern:
-                if not drive_group.placement.filter_matching_hosts(self.get_hosts):
-                    raise orchestrator.OrchestratorValidationError('failed to match')
+            if not drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None:
+                                                               [h.hostname for h in all_hosts]):
+                raise orchestrator.OrchestratorValidationError('failed to match')
+
         return self.get_hosts().then(run).then(
             on_complete=orchestrator.ProgressReference(
                 message='create_osds',
@@ -277,9 +278,9 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator):
         def run(all_hosts):
             # type: (List[orchestrator.HostSpec]) -> None
             drive_group.validate()
-            if drive_group.placement.host_pattern:
-                if not drive_group.placement.filter_matching_hosts(self.get_hosts):
-                    raise orchestrator.OrchestratorValidationError('failed to match')
+            if not drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None:
+                                                               [h.hostname for h in all_hosts]):
+                raise orchestrator.OrchestratorValidationError('failed to match')
         return self.get_hosts().then(run).then(
             on_complete=orchestrator.ProgressReference(
                 message='apply_drivesgroups',

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 9e05293d8f30e2b712359571afc52a227daf766b..627a7ca6a28dd6d72922a25b3796e72bf17d6b19 100644 (file)
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -2,7 +2,7 @@ import fnmatch
 import re
 from collections import namedtuple
 from functools import wraps
-from typing import Optional, Dict, Any, List, Union
+from typing import Optional, Dict, Any, List, Union, Callable
 
 import six
 
@@ -173,11 +173,23 @@ class PlacementSpec(object):
         # in the orchestrator backend.
         self.hosts = hosts
 
-    def pattern_matches_hosts(self, all_hosts):
-        # type: (List[str]) -> List[str]
-        if not self.host_pattern:
+    def filter_matching_hosts(self, _get_hosts_func: Callable) -> List[str]:
+        if self.hosts:
+            all_hosts = _get_hosts_func(label=None, as_hostspec=False)
+            return [h.hostname for h in self.hosts if h.hostname in all_hosts]
+        elif self.label:
+            return _get_hosts_func(label=self.label, as_hostspec=False)
+        elif self.host_pattern:
+            return fnmatch.filter(_get_hosts_func(label=None, as_hostspec=False), self.host_pattern)
+        else:
+            # This should be caught by the validation but needs to be here for
+            # get_host_selection_size
             return []
-        return fnmatch.filter(all_hosts, self.host_pattern)
+
+    def get_host_selection_size(self, _get_hosts_func):
+        if self.count:
+            return self.count
+        return len(self.filter_matching_hosts(_get_hosts_func))
 
     def pretty_str(self):
         kv = []

diff --git a/src/python-common/ceph/tests/test_drive_group.py b/src/python-common/ceph/tests/test_drive_group.py
index c0734ced82e39919cd86a16bbbf6ba0fac75f353..a34e93aaa27c9eccea2b94cef45edd484aa5563f 100644 (file)
--- a/src/python-common/ceph/tests/test_drive_group.py
+++ b/src/python-common/ceph/tests/test_drive_group.py
@@ -24,7 +24,7 @@ from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection, \
 ])
 def test_DriveGroup(test_input):
     dg = [DriveGroupSpec.from_json(inp) for inp in test_input][0]
-    assert dg.placement.pattern_matches_hosts(['hostname']) == ['hostname']
+    assert dg.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: ['hostname']) == ['hostname']
     assert dg.service_id == 'testing_drivegroup'
     assert all([isinstance(x, Device) for x in dg.data_devices.paths])
     assert dg.data_devices.paths[0].path == '/dev/sda'
@@ -53,7 +53,7 @@ def test_DriveGroup_fail(test_input):
 
 def test_drivegroup_pattern():
     dg = DriveGroupSpec(PlacementSpec(host_pattern='node[1-3]'), data_devices=DeviceSelection(all=True))
-    assert dg.placement.pattern_matches_hosts(['node{}'.format(i) for i in range(10)]) == ['node1', 'node2', 'node3']
+    assert dg.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: ['node{}'.format(i) for i in range(10)]) == ['node1', 'node2', 'node3']
 
 
 def test_drive_selection():