From d2aa0543d924d659fa0c5321b3fa9c607a44b5e3 Mon Sep 17 00:00:00 2001 From: Joshua Schmid Date: Tue, 12 May 2020 10:25:40 +0200 Subject: [PATCH] python-common: improve host matching funcs Signed-off-by: Joshua Schmid --- src/pybind/mgr/cephadm/module.py | 5 ++--- src/pybind/mgr/rook/module.py | 12 +++++----- src/pybind/mgr/test_orchestrator/module.py | 13 ++++++----- .../ceph/deployment/service_spec.py | 22 ++++++++++++++----- .../ceph/tests/test_drive_group.py | 4 ++-- 5 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index e252dffca79..c820ec1b5fb 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1510,14 +1510,13 @@ you may want to run: if not osdspecs: self.log.debug("No OSDSpecs found") return [] - # TODO: adapt this when we change patter_matches_hosts with https://github.com/ceph/ceph/pull/34860 - return sum([spec.placement.pattern_matches_hosts(self.cache.get_hosts()) for spec in osdspecs], []) + return sum([spec.placement.filter_matching_hosts(self._get_hosts) for spec in osdspecs], []) def resolve_osdspecs_for_host(self, host): matching_specs = [] self.log.debug(f"Finding OSDSpecs for host: <{host}>") for spec in self.spec_store.find('osd'): - if host in spec.placement.pattern_matches_hosts(self.cache.get_hosts()): + if host in spec.placement.filter_matching_hosts(self._get_hosts): self.log.debug(f"Found OSDSpecs for host: <{host}> -> <{spec}>") matching_specs.append(spec) return matching_specs diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 44228731ba0..4b12d8e781a 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -460,7 +460,6 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator): return self._service_add_decorate('RGW', spec, self.rook_cluster.apply_objectstore) - def apply_nfs(self, spec): # type: (NFSServiceSpec) -> RookCompletion num = spec.placement.count @@ -478,7 +477,7 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator): ) def create_osds(self, drive_group): - # type: (DriveGroupSpec) -> orchestrator.Completion + # type: (DriveGroupSpec) -> RookCompletion """ Creates OSDs from a drive group specification. $: ceph orch osd create -i @@ -492,9 +491,10 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator): if drive_group.data_directories: targets += drive_group.data_directories - def execute(): - # type: () -> orchestrator.Completion - matching_hosts = drive_group.placement.filter_matching_hosts(self.get_hosts) + def execute(all_hosts_): + # type: (List[orchestrator.HostSpec]) -> orchestrator.Completion + all_hosts = [h.hostname for h in all_hosts_] + matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts) assert len(matching_hosts) == 1 @@ -549,7 +549,7 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator): return found is not None - c = execute() + c = self.get_hosts().then(execute) return c def blink_device_light(self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> RookCompletion: diff --git a/src/pybind/mgr/test_orchestrator/module.py b/src/pybind/mgr/test_orchestrator/module.py index 7b464290ac4..d7ef5d5bcc7 100644 --- a/src/pybind/mgr/test_orchestrator/module.py +++ b/src/pybind/mgr/test_orchestrator/module.py @@ -258,9 +258,10 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator): def run(all_hosts): # type: (List[orchestrator.HostSpec]) -> None drive_group.validate() - if drive_group.placement.host_pattern: - if not drive_group.placement.filter_matching_hosts(self.get_hosts): - raise orchestrator.OrchestratorValidationError('failed to match') + if not drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: + [h.hostname for h in all_hosts]): + raise orchestrator.OrchestratorValidationError('failed to match') + return self.get_hosts().then(run).then( on_complete=orchestrator.ProgressReference( message='create_osds', @@ -275,9 +276,9 @@ class TestOrchestrator(MgrModule, orchestrator.Orchestrator): def run(all_hosts): # type: (List[orchestrator.HostSpec]) -> None drive_group.validate() - if drive_group.placement.host_pattern: - if not drive_group.placement.filter_matching_hosts(self.get_hosts): - raise orchestrator.OrchestratorValidationError('failed to match') + if not drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: + [h.hostname for h in all_hosts]): + raise orchestrator.OrchestratorValidationError('failed to match') return self.get_hosts().then(run).then( on_complete=orchestrator.ProgressReference( message='apply_drivesgroups', diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 9e05293d8f3..627a7ca6a28 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -2,7 +2,7 @@ import fnmatch import re from collections import namedtuple from functools import wraps -from typing import Optional, Dict, Any, List, Union +from typing import Optional, Dict, Any, List, Union, Callable import six @@ -173,11 +173,23 @@ class PlacementSpec(object): # in the orchestrator backend. self.hosts = hosts - def pattern_matches_hosts(self, all_hosts): - # type: (List[str]) -> List[str] - if not self.host_pattern: + def filter_matching_hosts(self, _get_hosts_func: Callable) -> List[str]: + if self.hosts: + all_hosts = _get_hosts_func(label=None, as_hostspec=False) + return [h.hostname for h in self.hosts if h.hostname in all_hosts] + elif self.label: + return _get_hosts_func(label=self.label, as_hostspec=False) + elif self.host_pattern: + return fnmatch.filter(_get_hosts_func(label=None, as_hostspec=False), self.host_pattern) + else: + # This should be caught by the validation but needs to be here for + # get_host_selection_size return [] - return fnmatch.filter(all_hosts, self.host_pattern) + + def get_host_selection_size(self, _get_hosts_func): + if self.count: + return self.count + return len(self.filter_matching_hosts(_get_hosts_func)) def pretty_str(self): kv = [] diff --git a/src/python-common/ceph/tests/test_drive_group.py b/src/python-common/ceph/tests/test_drive_group.py index c0734ced82e..a34e93aaa27 100644 --- a/src/python-common/ceph/tests/test_drive_group.py +++ b/src/python-common/ceph/tests/test_drive_group.py @@ -24,7 +24,7 @@ from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection, \ ]) def test_DriveGroup(test_input): dg = [DriveGroupSpec.from_json(inp) for inp in test_input][0] - assert dg.placement.pattern_matches_hosts(['hostname']) == ['hostname'] + assert dg.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: ['hostname']) == ['hostname'] assert dg.service_id == 'testing_drivegroup' assert all([isinstance(x, Device) for x in dg.data_devices.paths]) assert dg.data_devices.paths[0].path == '/dev/sda' @@ -53,7 +53,7 @@ def test_DriveGroup_fail(test_input): def test_drivegroup_pattern(): dg = DriveGroupSpec(PlacementSpec(host_pattern='node[1-3]'), data_devices=DeviceSelection(all=True)) - assert dg.placement.pattern_matches_hosts(['node{}'.format(i) for i in range(10)]) == ['node1', 'node2', 'node3'] + assert dg.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: ['node{}'.format(i) for i in range(10)]) == ['node1', 'node2', 'node3'] def test_drive_selection(): -- 2.47.3