import pytest
from ceph.deployment.hostspec import HostSpec
-from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, IngressSpec
+from ceph.deployment.service_spec import (
+ ServiceSpec,
+ PlacementSpec,
+ IngressSpec,
+ PatternType,
+ HostPattern,
+)
from ceph.deployment.hostspec import SpecValidationError
from cephadm.module import HostAssignment
).place()
assert sorted([h.hostname for h in to_add]) in expected_add
assert sorted([h.name() for h in to_remove]) in expected_remove
+
+
+class RegexHostPatternTest(NamedTuple):
+ service_type: str
+ placement: PlacementSpec
+ hosts: List[str]
+ expected_add: List[List[str]]
+
+
+@pytest.mark.parametrize("service_type,placement,hosts,expected_add",
+ [
+ RegexHostPatternTest(
+ 'crash',
+ PlacementSpec(host_pattern=HostPattern(pattern='host1|host3', pattern_type=PatternType.regex)),
+ 'host1 host2 host3 host4'.split(),
+ ['host1', 'host3'],
+ ),
+ RegexHostPatternTest(
+ 'crash',
+ PlacementSpec(host_pattern=HostPattern(pattern='host[2-4]', pattern_type=PatternType.regex)),
+ 'host1 host2 host3 host4'.split(),
+ ['host2', 'host3', 'host4'],
+ ),
+ ])
+def test_placement_regex_host_pattern(service_type, placement, hosts, expected_add):
+ spec = ServiceSpec(service_type=service_type,
+ service_id='test',
+ placement=placement)
+
+ host_specs = [HostSpec(h) for h in hosts]
+
+ hosts, to_add, to_remove = HostAssignment(
+ spec=spec,
+ hosts=host_specs,
+ unreachable_hosts=[],
+ draining_hosts=[],
+ daemons=[],
+ ).place()
+ assert sorted([h.hostname for h in to_add]) == expected_add
assert_valid_host(self.hostname)
+HostPatternType = Union[str, None, Dict[str, Union[str, bool, None]], "HostPattern"]
+
+
+class PatternType(enum.Enum):
+ fnmatch = 'fnmatch'
+ regex = 'regex'
+
+
+class HostPattern():
+ def __init__(self,
+ pattern: Optional[str] = None,
+ pattern_type: PatternType = PatternType.fnmatch) -> None:
+ self.pattern: Optional[str] = pattern
+ self.pattern_type: PatternType = pattern_type
+ self.compiled_regex = None
+ if self.pattern_type == PatternType.regex and self.pattern:
+ self.compiled_regex = re.compile(self.pattern)
+
+ def filter_hosts(self, hosts: List[str]) -> List[str]:
+ if not self.pattern:
+ return []
+ if not self.pattern_type or self.pattern_type == PatternType.fnmatch:
+ return fnmatch.filter(hosts, self.pattern)
+ elif self.pattern_type == PatternType.regex:
+ if not self.compiled_regex:
+ self.compiled_regex = re.compile(self.pattern)
+ return [h for h in hosts if re.match(self.compiled_regex, h)]
+ raise SpecValidationError(f'Got unexpected pattern_type: {self.pattern_type}')
+
+ @classmethod
+ def to_host_pattern(cls, arg: HostPatternType) -> "HostPattern":
+ if arg is None:
+ return cls()
+ elif isinstance(arg, str):
+ return cls(arg)
+ elif isinstance(arg, cls):
+ return arg
+ elif isinstance(arg, dict):
+ if 'pattern' not in arg:
+ raise SpecValidationError("Got dict for host pattern "
+ f"with no pattern field: {arg}")
+ pattern = arg['pattern']
+ if not pattern:
+ raise SpecValidationError("Got dict for host pattern"
+ f"with empty pattern: {arg}")
+ assert isinstance(pattern, str)
+ if 'pattern_type' in arg:
+ pattern_type = arg['pattern_type']
+ if not pattern_type or pattern_type == 'fnmatch':
+ return cls(pattern, pattern_type=PatternType.fnmatch)
+ elif pattern_type == 'regex':
+ return cls(pattern, pattern_type=PatternType.regex)
+ else:
+ raise SpecValidationError("Got dict for host pattern "
+ f"with unknown pattern type: {arg}")
+ return cls(pattern)
+ raise SpecValidationError(f"Cannot convert {type(arg)} object to HostPattern")
+
+ def __eq__(self, other: Any) -> bool:
+ try:
+ other_hp = self.to_host_pattern(other)
+ except SpecValidationError:
+ return False
+ return self.pattern == other_hp.pattern and self.pattern_type == other_hp.pattern_type
+
+ def pretty_str(self) -> str:
+ # Placement specs must be able to be converted between the Python object
+ # representation and a pretty str both ways. So we need a corresponding
+ # function for HostPattern to convert it to a pretty str that we can
+ # convert back later.
+ res = self.pattern if self.pattern else ''
+ if self.pattern_type == PatternType.regex:
+ res = 'regex:' + res
+ return res
+
+ @classmethod
+ def from_pretty_str(cls, val: str) -> "HostPattern":
+ if 'regex:' in val:
+ return cls(val[6:], pattern_type=PatternType.regex)
+ else:
+ return cls(val)
+
+ def __repr__(self) -> str:
+ return f'HostPattern(pattern=\'{self.pattern}\', pattern_type={str(self.pattern_type)})'
+
+ def to_json(self) -> Union[str, Dict[str, Any], None]:
+ if self.pattern_type and self.pattern_type != PatternType.fnmatch:
+ return {
+ 'pattern': self.pattern,
+ 'pattern_type': self.pattern_type.name
+ }
+ return self.pattern
+
+ @classmethod
+ def from_json(self, val: Dict[str, Any]) -> "HostPattern":
+ return self.to_host_pattern(val)
+
+ def __bool__(self) -> bool:
+ if self.pattern:
+ return True
+ return False
+
+
class PlacementSpec(object):
"""
For APIs that need to specify a host subset
"""
def __init__(self,
- label=None, # type: Optional[str]
- hosts=None, # type: Union[List[str],List[HostPlacementSpec], None]
- count=None, # type: Optional[int]
- count_per_host=None, # type: Optional[int]
- host_pattern=None, # type: Optional[str]
+ label: Optional[str] = None,
+ hosts: Union[List[str], List[HostPlacementSpec], None] = None,
+ count: Optional[int] = None,
+ count_per_host: Optional[int] = None,
+ host_pattern: HostPatternType = None,
):
# type: (...) -> None
self.label = label
self.count_per_host = count_per_host # type: Optional[int]
#: fnmatch patterns to select hosts. Can also be a single host.
- self.host_pattern = host_pattern # type: Optional[str]
+ self.host_pattern: HostPattern = HostPattern.to_host_pattern(host_pattern)
self.validate()
return [hs.hostname for hs in hostspecs if self.label in hs.labels]
all_hosts = [hs.hostname for hs in hostspecs]
if self.host_pattern:
- return fnmatch.filter(all_hosts, self.host_pattern)
+ return self.host_pattern.filter_hosts(all_hosts)
return all_hosts
def get_target_count(self, hostspecs: Iterable[HostSpec]) -> int:
if self.label:
kv.append('label:%s' % self.label)
if self.host_pattern:
- kv.append(self.host_pattern)
+ kv.append(self.host_pattern.pretty_str())
return ';'.join(kv)
def __repr__(self) -> str:
if self.count_per_host:
r['count_per_host'] = self.count_per_host
if self.host_pattern:
- r['host_pattern'] = self.host_pattern
+ r['host_pattern'] = self.host_pattern.to_json()
return r
def validate(self) -> None:
"count-per-host cannot be combined explicit placement with names or networks"
)
if self.host_pattern:
- if not isinstance(self.host_pattern, str):
- raise SpecValidationError('host_pattern must be of type string')
+ # if we got an invalid type for the host_pattern, it would have
+ # triggered a SpecValidationError when attemptying to convert it
+ # to a HostPattern type, so no type checking is needed here.
if self.hosts:
raise SpecValidationError('cannot combine host patterns and hosts')
>>> PlacementSpec.from_string('3 label:mon')
PlacementSpec(count=3, label='mon')
- fnmatch is also supported:
+ You can specify a regex to match with `regex:<regex>`
+
+ >>> PlacementSpec.from_string('regex:Foo[0-9]|Bar[0-9]')
+ PlacementSpec(host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', \
+pattern_type=PatternType.regex))
+
+ fnmatch is the default for a single string if "regex:" is not provided:
>>> PlacementSpec.from_string('data[1-3]')
- PlacementSpec(host_pattern='data[1-3]')
+ PlacementSpec(host_pattern=HostPattern(pattern='data[1-3]', \
+pattern_type=PatternType.fnmatch))
>>> PlacementSpec.from_string(None)
PlacementSpec()
advanced_hostspecs = [h for h in strings if
(':' in h or '=' in h or not any(c in '[]?*:=' for c in h)) and
- 'label:' not in h]
+ 'label:' not in h and
+ 'regex:' not in h]
for a_h in advanced_hostspecs:
strings.remove(a_h)
label = labels[0][6:] if labels else None
host_patterns = strings
+ host_pattern: Optional[HostPattern] = None
if len(host_patterns) > 1:
raise SpecValidationError(
'more than one host pattern provided: {}'.format(host_patterns))
+ if host_patterns:
+ # host_patterns is a list not > 1, and not empty, so we should
+ # be guaranteed just a single string here
+ host_pattern = HostPattern.from_pretty_str(host_patterns[0])
ps = PlacementSpec(count=count,
count_per_host=count_per_host,
hosts=advanced_hostspecs,
label=label,
- host_pattern=host_patterns[0] if host_patterns else None)
+ host_pattern=host_pattern)
return ps
('2 host1 host2', "PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])"),
('label:foo', "PlacementSpec(label='foo')"),
('3 label:foo', "PlacementSpec(count=3, label='foo')"),
- ('*', "PlacementSpec(host_pattern='*')"),
- ('3 data[1-3]', "PlacementSpec(count=3, host_pattern='data[1-3]')"),
- ('3 data?', "PlacementSpec(count=3, host_pattern='data?')"),
- ('3 data*', "PlacementSpec(count=3, host_pattern='data*')"),
+ ('*', "PlacementSpec(host_pattern=HostPattern(pattern='*', pattern_type=PatternType.fnmatch))"),
+ ('3 data[1-3]', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data[1-3]', pattern_type=PatternType.fnmatch))"),
+ ('3 data?', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data?', pattern_type=PatternType.fnmatch))"),
+ ('3 data*', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data*', pattern_type=PatternType.fnmatch))"),
("count-per-host:4 label:foo", "PlacementSpec(count_per_host=4, label='foo')"),
+ ('regex:Foo[0-9]|Bar[0-9]', "PlacementSpec(host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', pattern_type=PatternType.regex))"),
+ ('3 regex:Foo[0-9]|Bar[0-9]', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', pattern_type=PatternType.regex))"),
])
def test_parse_placement_specs(test_input, expected):
ret = PlacementSpec.from_string(test_input)
("host=a host*"),
("host=a label:wrong"),
("host? host*"),
+ ("host? regex:host*"),
+ ("regex:host? host*"),
+ ("regex:host? regex:host*"),
('host=a count-per-host:0'),
('host=a count-per-host:-10'),
('count:2 count-per-host:1'),
host_pattern: '*'
unmanaged: true
---
+service_type: crash
+service_name: crash
+placement:
+ host_pattern:
+ pattern: Foo[0-9]|Bar[0-9]
+ pattern_type: regex
+---
service_type: rgw
service_id: default-rgw-realm.eu-central-1.1
service_name: rgw.default-rgw-realm.eu-central-1.1
projects / ceph.git / commitdiff