From f27790b057db667c48b1840472046db3d6d9c5f1 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 3 Oct 2023 19:06:10 -0400 Subject: [PATCH] mgr/cephadm: support for regex based host patterns For example, with hosts vm-00, vm-01, and vm-02 I was able to provide the placement service_type: node-exporter service_name: node-exporter placement: host_pattern: pattern: vm-00|vm-02 pattern_type: regex and it placed the node-exporter daemons on vm-00 and vm-02 but not vm-01. Obviously there are more advanced scenarios that justify this than listing two hosts, but using "|" as an OR like that is an example of something you can't do with the fnmatch version of the host pattern Signed-off-by: Adam King --- .../mgr/cephadm/tests/test_scheduling.py | 47 +++++- src/pybind/mgr/rook/rook_cluster.py | 11 +- .../ceph/deployment/service_spec.py | 147 ++++++++++++++++-- .../ceph/tests/test_service_spec.py | 20 ++- 4 files changed, 203 insertions(+), 22 deletions(-) diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py index 067cd5028a2cf..b307cd9d34ddc 100644 --- a/src/pybind/mgr/cephadm/tests/test_scheduling.py +++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py @@ -6,7 +6,13 @@ from typing import NamedTuple, List, Dict, Optional import pytest from ceph.deployment.hostspec import HostSpec -from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, IngressSpec +from ceph.deployment.service_spec import ( + ServiceSpec, + PlacementSpec, + IngressSpec, + PatternType, + HostPattern, +) from ceph.deployment.hostspec import SpecValidationError from cephadm.module import HostAssignment @@ -1697,3 +1703,42 @@ def test_drain_from_explict_placement(service_type, placement, hosts, maintenanc ).place() assert sorted([h.hostname for h in to_add]) in expected_add assert sorted([h.name() for h in to_remove]) in expected_remove + + +class RegexHostPatternTest(NamedTuple): + service_type: str + placement: PlacementSpec + hosts: List[str] + expected_add: List[List[str]] + + +@pytest.mark.parametrize("service_type,placement,hosts,expected_add", + [ + RegexHostPatternTest( + 'crash', + PlacementSpec(host_pattern=HostPattern(pattern='host1|host3', pattern_type=PatternType.regex)), + 'host1 host2 host3 host4'.split(), + ['host1', 'host3'], + ), + RegexHostPatternTest( + 'crash', + PlacementSpec(host_pattern=HostPattern(pattern='host[2-4]', pattern_type=PatternType.regex)), + 'host1 host2 host3 host4'.split(), + ['host2', 'host3', 'host4'], + ), + ]) +def test_placement_regex_host_pattern(service_type, placement, hosts, expected_add): + spec = ServiceSpec(service_type=service_type, + service_id='test', + placement=placement) + + host_specs = [HostSpec(h) for h in hosts] + + hosts, to_add, to_remove = HostAssignment( + spec=spec, + hosts=host_specs, + unreachable_hosts=[], + draining_hosts=[], + daemons=[], + ).place() + assert sorted([h.hostname for h in to_add]) == expected_add diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index 5c7c9fc0477f9..70581e6051da2 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -24,7 +24,14 @@ from urllib3.exceptions import ProtocolError from ceph.deployment.inventory import Device from ceph.deployment.drive_group import DriveGroupSpec -from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, PlacementSpec, HostPlacementSpec +from ceph.deployment.service_spec import ( + ServiceSpec, + NFSServiceSpec, + RGWSpec, + PlacementSpec, + HostPlacementSpec, + HostPattern, +) from ceph.utils import datetime_now from ceph.deployment.drive_selection.matchers import SizeMatcher from nfs.cluster import create_ganesha_pool @@ -1585,7 +1592,7 @@ def node_selector_to_placement_spec(node_selector: ccl.NodeSelectorTermsItem) -> res.label = expression.key.split('/')[1] elif expression.key == "kubernetes.io/hostname": if expression.operator == "Exists": - res.host_pattern = "*" + res.host_pattern = HostPattern("*") elif expression.operator == "In": res.hosts = [HostPlacementSpec(hostname=value, network='', name='')for value in expression.values] return res diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 4181ee2563e49..28f9a51a8a52d 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -140,17 +140,120 @@ class HostPlacementSpec(NamedTuple): assert_valid_host(self.hostname) +HostPatternType = Union[str, None, Dict[str, Union[str, bool, None]], "HostPattern"] + + +class PatternType(enum.Enum): + fnmatch = 'fnmatch' + regex = 'regex' + + +class HostPattern(): + def __init__(self, + pattern: Optional[str] = None, + pattern_type: PatternType = PatternType.fnmatch) -> None: + self.pattern: Optional[str] = pattern + self.pattern_type: PatternType = pattern_type + self.compiled_regex = None + if self.pattern_type == PatternType.regex and self.pattern: + self.compiled_regex = re.compile(self.pattern) + + def filter_hosts(self, hosts: List[str]) -> List[str]: + if not self.pattern: + return [] + if not self.pattern_type or self.pattern_type == PatternType.fnmatch: + return fnmatch.filter(hosts, self.pattern) + elif self.pattern_type == PatternType.regex: + if not self.compiled_regex: + self.compiled_regex = re.compile(self.pattern) + return [h for h in hosts if re.match(self.compiled_regex, h)] + raise SpecValidationError(f'Got unexpected pattern_type: {self.pattern_type}') + + @classmethod + def to_host_pattern(cls, arg: HostPatternType) -> "HostPattern": + if arg is None: + return cls() + elif isinstance(arg, str): + return cls(arg) + elif isinstance(arg, cls): + return arg + elif isinstance(arg, dict): + if 'pattern' not in arg: + raise SpecValidationError("Got dict for host pattern " + f"with no pattern field: {arg}") + pattern = arg['pattern'] + if not pattern: + raise SpecValidationError("Got dict for host pattern" + f"with empty pattern: {arg}") + assert isinstance(pattern, str) + if 'pattern_type' in arg: + pattern_type = arg['pattern_type'] + if not pattern_type or pattern_type == 'fnmatch': + return cls(pattern, pattern_type=PatternType.fnmatch) + elif pattern_type == 'regex': + return cls(pattern, pattern_type=PatternType.regex) + else: + raise SpecValidationError("Got dict for host pattern " + f"with unknown pattern type: {arg}") + return cls(pattern) + raise SpecValidationError(f"Cannot convert {type(arg)} object to HostPattern") + + def __eq__(self, other: Any) -> bool: + try: + other_hp = self.to_host_pattern(other) + except SpecValidationError: + return False + return self.pattern == other_hp.pattern and self.pattern_type == other_hp.pattern_type + + def pretty_str(self) -> str: + # Placement specs must be able to be converted between the Python object + # representation and a pretty str both ways. So we need a corresponding + # function for HostPattern to convert it to a pretty str that we can + # convert back later. + res = self.pattern if self.pattern else '' + if self.pattern_type == PatternType.regex: + res = 'regex:' + res + return res + + @classmethod + def from_pretty_str(cls, val: str) -> "HostPattern": + if 'regex:' in val: + return cls(val[6:], pattern_type=PatternType.regex) + else: + return cls(val) + + def __repr__(self) -> str: + return f'HostPattern(pattern=\'{self.pattern}\', pattern_type={str(self.pattern_type)})' + + def to_json(self) -> Union[str, Dict[str, Any], None]: + if self.pattern_type and self.pattern_type != PatternType.fnmatch: + return { + 'pattern': self.pattern, + 'pattern_type': self.pattern_type.name + } + return self.pattern + + @classmethod + def from_json(self, val: Dict[str, Any]) -> "HostPattern": + return self.to_host_pattern(val) + + def __bool__(self) -> bool: + if self.pattern: + return True + return False + + class PlacementSpec(object): """ For APIs that need to specify a host subset """ def __init__(self, - label=None, # type: Optional[str] - hosts=None, # type: Union[List[str],List[HostPlacementSpec], None] - count=None, # type: Optional[int] - count_per_host=None, # type: Optional[int] - host_pattern=None, # type: Optional[str] + label: Optional[str] = None, + hosts: Union[List[str], List[HostPlacementSpec], None] = None, + count: Optional[int] = None, + count_per_host: Optional[int] = None, + host_pattern: HostPatternType = None, ): # type: (...) -> None self.label = label @@ -163,7 +266,7 @@ class PlacementSpec(object): self.count_per_host = count_per_host # type: Optional[int] #: fnmatch patterns to select hosts. Can also be a single host. - self.host_pattern = host_pattern # type: Optional[str] + self.host_pattern: HostPattern = HostPattern.to_host_pattern(host_pattern) self.validate() @@ -206,7 +309,7 @@ class PlacementSpec(object): return [hs.hostname for hs in hostspecs if self.label in hs.labels] all_hosts = [hs.hostname for hs in hostspecs] if self.host_pattern: - return fnmatch.filter(all_hosts, self.host_pattern) + return self.host_pattern.filter_hosts(all_hosts) return all_hosts def get_target_count(self, hostspecs: Iterable[HostSpec]) -> int: @@ -230,7 +333,7 @@ class PlacementSpec(object): if self.label: kv.append('label:%s' % self.label) if self.host_pattern: - kv.append(self.host_pattern) + kv.append(self.host_pattern.pretty_str()) return ';'.join(kv) def __repr__(self) -> str: @@ -271,7 +374,7 @@ class PlacementSpec(object): if self.count_per_host: r['count_per_host'] = self.count_per_host if self.host_pattern: - r['host_pattern'] = self.host_pattern + r['host_pattern'] = self.host_pattern.to_json() return r def validate(self) -> None: @@ -315,8 +418,9 @@ class PlacementSpec(object): "count-per-host cannot be combined explicit placement with names or networks" ) if self.host_pattern: - if not isinstance(self.host_pattern, str): - raise SpecValidationError('host_pattern must be of type string') + # if we got an invalid type for the host_pattern, it would have + # triggered a SpecValidationError when attemptying to convert it + # to a HostPattern type, so no type checking is needed here. if self.hosts: raise SpecValidationError('cannot combine host patterns and hosts') @@ -354,10 +458,17 @@ tPlacementSpec(hostname='host2', network='', name='')]) >>> PlacementSpec.from_string('3 label:mon') PlacementSpec(count=3, label='mon') - fnmatch is also supported: + You can specify a regex to match with `regex:` + + >>> PlacementSpec.from_string('regex:Foo[0-9]|Bar[0-9]') + PlacementSpec(host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', \ +pattern_type=PatternType.regex)) + + fnmatch is the default for a single string if "regex:" is not provided: >>> PlacementSpec.from_string('data[1-3]') - PlacementSpec(host_pattern='data[1-3]') + PlacementSpec(host_pattern=HostPattern(pattern='data[1-3]', \ +pattern_type=PatternType.fnmatch)) >>> PlacementSpec.from_string(None) PlacementSpec() @@ -407,7 +518,8 @@ tPlacementSpec(hostname='host2', network='', name='')]) advanced_hostspecs = [h for h in strings if (':' in h or '=' in h or not any(c in '[]?*:=' for c in h)) and - 'label:' not in h] + 'label:' not in h and + 'regex:' not in h] for a_h in advanced_hostspecs: strings.remove(a_h) @@ -419,15 +531,20 @@ tPlacementSpec(hostname='host2', network='', name='')]) label = labels[0][6:] if labels else None host_patterns = strings + host_pattern: Optional[HostPattern] = None if len(host_patterns) > 1: raise SpecValidationError( 'more than one host pattern provided: {}'.format(host_patterns)) + if host_patterns: + # host_patterns is a list not > 1, and not empty, so we should + # be guaranteed just a single string here + host_pattern = HostPattern.from_pretty_str(host_patterns[0]) ps = PlacementSpec(count=count, count_per_host=count_per_host, hosts=advanced_hostspecs, label=label, - host_pattern=host_patterns[0] if host_patterns else None) + host_pattern=host_pattern) return ps diff --git a/src/python-common/ceph/tests/test_service_spec.py b/src/python-common/ceph/tests/test_service_spec.py index 502057f5ca3b6..9a55b0a813598 100644 --- a/src/python-common/ceph/tests/test_service_spec.py +++ b/src/python-common/ceph/tests/test_service_spec.py @@ -144,11 +144,13 @@ def test_apply_prometheus(spec: PrometheusSpec, raise_exception: bool, msg: str) ('2 host1 host2', "PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])"), ('label:foo', "PlacementSpec(label='foo')"), ('3 label:foo', "PlacementSpec(count=3, label='foo')"), - ('*', "PlacementSpec(host_pattern='*')"), - ('3 data[1-3]', "PlacementSpec(count=3, host_pattern='data[1-3]')"), - ('3 data?', "PlacementSpec(count=3, host_pattern='data?')"), - ('3 data*', "PlacementSpec(count=3, host_pattern='data*')"), + ('*', "PlacementSpec(host_pattern=HostPattern(pattern='*', pattern_type=PatternType.fnmatch))"), + ('3 data[1-3]', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data[1-3]', pattern_type=PatternType.fnmatch))"), + ('3 data?', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data?', pattern_type=PatternType.fnmatch))"), + ('3 data*', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data*', pattern_type=PatternType.fnmatch))"), ("count-per-host:4 label:foo", "PlacementSpec(count_per_host=4, label='foo')"), + ('regex:Foo[0-9]|Bar[0-9]', "PlacementSpec(host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', pattern_type=PatternType.regex))"), + ('3 regex:Foo[0-9]|Bar[0-9]', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', pattern_type=PatternType.regex))"), ]) def test_parse_placement_specs(test_input, expected): ret = PlacementSpec.from_string(test_input) @@ -161,6 +163,9 @@ def test_parse_placement_specs(test_input, expected): ("host=a host*"), ("host=a label:wrong"), ("host? host*"), + ("host? regex:host*"), + ("regex:host? host*"), + ("regex:host? regex:host*"), ('host=a count-per-host:0'), ('host=a count-per-host:-10'), ('count:2 count-per-host:1'), @@ -313,6 +318,13 @@ placement: host_pattern: '*' unmanaged: true --- +service_type: crash +service_name: crash +placement: + host_pattern: + pattern: Foo[0-9]|Bar[0-9] + pattern_type: regex +--- service_type: rgw service_id: default-rgw-realm.eu-central-1.1 service_name: rgw.default-rgw-realm.eu-central-1.1 -- 2.39.5