From: Adam King Date: Fri, 29 Jul 2022 20:10:09 +0000 (-0400) Subject: mgr/cephadm: fix handling of draining hosts with explicit placement specs X-Git-Tag: v17.2.6~98^2~59^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=ccdc075ad97d3ed5e58cb15713bd1b548e6e61ca;p=ceph.git mgr/cephadm: fix handling of draining hosts with explicit placement specs Basically, if you have a placement that explicitly defines the hosts to place on, and then add _no_schedule label to one of the hosts (which should cause all daemons to be removed from the host) cpehadm will simply fail to apply the spec, saying the host with the _no_schedule label is "Unknown". This is due to the fact that we remove hosts with the _no_schedule label from the pool of hosts the scheduler has to work with entirely. If we also provide the scheduler with a list of currently draining hosts, it can handle this better and the daemon can be drained off the host as expected. Fixes: https://tracker.ceph.com/issues/56972 Signed-off-by: Adam King (cherry picked from commit 7e8c07a3dd998dd3745b7f36919a21ca613484e4) --- diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py index 7a88258f002dc..bf0dbb534101e 100644 --- a/src/pybind/mgr/cephadm/inventory.py +++ b/src/pybind/mgr/cephadm/inventory.py @@ -936,6 +936,15 @@ class HostCache(): h for h in self.mgr.inventory.all_specs() if '_no_schedule' not in h.labels ] + def get_draining_hosts(self) -> List[HostSpec]: + """ + Returns all hosts that have _no_schedule label and therefore should have + no daemons placed on them, but are potentially still reachable + """ + return [ + h for h in self.mgr.inventory.all_specs() if '_no_schedule' in h.labels + ] + def get_unreachable_hosts(self) -> List[HostSpec]: """ Return all hosts that are offline or in maintenance mode. diff --git a/src/pybind/mgr/cephadm/migrations.py b/src/pybind/mgr/cephadm/migrations.py index 672a895bc14c8..69f39cb910770 100644 --- a/src/pybind/mgr/cephadm/migrations.py +++ b/src/pybind/mgr/cephadm/migrations.py @@ -112,6 +112,7 @@ class Migrations: spec=spec, hosts=self.mgr.inventory.all_specs(), unreachable_hosts=self.mgr.cache.get_unreachable_hosts(), + draining_hosts=self.mgr.cache.get_draining_hosts(), daemons=existing_daemons, ).place() diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index e06ff1359e5d8..118670d287007 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2538,6 +2538,7 @@ Then run the following: spec=spec, hosts=self.cache.get_schedulable_hosts(), unreachable_hosts=self.cache.get_unreachable_hosts(), + draining_hosts=self.cache.get_draining_hosts(), networks=self.cache.networks, daemons=self.cache.get_daemons_by_service(spec.service_name()), allow_colo=svc.allow_colo(), @@ -2616,6 +2617,7 @@ Then run the following: spec=spec, hosts=self.inventory.all_specs(), # All hosts, even those without daemon refresh unreachable_hosts=self.cache.get_unreachable_hosts(), + draining_hosts=self.cache.get_draining_hosts(), networks=self.cache.networks, daemons=self.cache.get_daemons_by_service(spec.service_name()), allow_colo=self.cephadm_services[spec.service_type].allow_colo(), diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py index 5002ec6e5060e..c80b6781869da 100644 --- a/src/pybind/mgr/cephadm/schedule.py +++ b/src/pybind/mgr/cephadm/schedule.py @@ -143,6 +143,7 @@ class HostAssignment(object): spec, # type: ServiceSpec hosts: List[orchestrator.HostSpec], unreachable_hosts: List[orchestrator.HostSpec], + draining_hosts: List[orchestrator.HostSpec], daemons: List[orchestrator.DaemonDescription], networks: Dict[str, Dict[str, Dict[str, List[str]]]] = {}, filter_new_host=None, # type: Optional[Callable[[str],bool]] @@ -156,6 +157,7 @@ class HostAssignment(object): self.primary_daemon_type = primary_daemon_type or spec.service_type self.hosts: List[orchestrator.HostSpec] = hosts self.unreachable_hosts: List[orchestrator.HostSpec] = unreachable_hosts + self.draining_hosts: List[orchestrator.HostSpec] = draining_hosts self.filter_new_host = filter_new_host self.service_name = spec.service_name() self.daemons = daemons @@ -189,7 +191,8 @@ class HostAssignment(object): if self.spec.placement.hosts: explicit_hostnames = {h.hostname for h in self.spec.placement.hosts} - unknown_hosts = explicit_hostnames.difference(set(self.get_hostnames())) + known_hosts = self.get_hostnames() + [h.hostname for h in self.draining_hosts] + unknown_hosts = explicit_hostnames.difference(set(known_hosts)) if unknown_hosts: raise OrchestratorValidationError( f'Cannot place {self.spec.one_line_str()} on {", ".join(sorted(unknown_hosts))}: Unknown hosts') @@ -371,7 +374,7 @@ class HostAssignment(object): DaemonPlacement(daemon_type=self.primary_daemon_type, hostname=h.hostname, network=h.network, name=h.name, ports=self.ports_start) - for h in self.spec.placement.hosts + for h in self.spec.placement.hosts if h.hostname not in [dh.hostname for dh in self.draining_hosts] ] elif self.spec.placement.label: ls = [ diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index 2f26ca70900f7..88d7b2083779f 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -627,6 +627,7 @@ class CephadmServe: hosts=self.mgr.cache.get_non_draining_hosts() if spec.service_name( ) == 'agent' else self.mgr.cache.get_schedulable_hosts(), unreachable_hosts=self.mgr.cache.get_unreachable_hosts(), + draining_hosts=self.mgr.cache.get_draining_hosts(), daemons=daemons, networks=self.mgr.cache.networks, filter_new_host=( @@ -1005,6 +1006,7 @@ class CephadmServe: spec=ServiceSpec('mon', placement=pspec), hosts=self.mgr.cache.get_schedulable_hosts(), unreachable_hosts=self.mgr.cache.get_unreachable_hosts(), + draining_hosts=self.mgr.cache.get_draining_hosts(), daemons=[], networks=self.mgr.cache.networks, ) @@ -1035,6 +1037,7 @@ class CephadmServe: spec=ServiceSpec('mon', placement=ks.placement), hosts=self.mgr.cache.get_schedulable_hosts(), unreachable_hosts=self.mgr.cache.get_unreachable_hosts(), + draining_hosts=self.mgr.cache.get_draining_hosts(), daemons=[], networks=self.mgr.cache.networks, ) diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py index 2454dc0d1ad0b..ffdad8e94068a 100644 --- a/src/pybind/mgr/cephadm/tests/test_scheduling.py +++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py @@ -133,6 +133,7 @@ def run_scheduler_test(results, mk_spec, hosts, daemons, key_elems): spec=spec, hosts=hosts, unreachable_hosts=[], + draining_hosts=[], daemons=daemons, ).place() if isinstance(host_res, list): @@ -149,6 +150,7 @@ def run_scheduler_test(results, mk_spec, hosts, daemons, key_elems): spec=spec, hosts=hosts, unreachable_hosts=[], + draining_hosts=[], daemons=daemons ).place() @@ -841,6 +843,7 @@ def test_node_assignment(service_type, placement, hosts, daemons, rank_map, post spec=spec, hosts=[HostSpec(h, labels=['foo']) for h in hosts], unreachable_hosts=[], + draining_hosts=[], daemons=daemons, allow_colo=allow_colo, rank_map=rank_map, @@ -935,6 +938,7 @@ def test_node_assignment_random_shuffle(service_type, placement, available_hosts spec=spec, hosts=[HostSpec(h, labels=['foo']) for h in available_hosts], unreachable_hosts=[], + draining_hosts=[], daemons=[], allow_colo=allow_colo, ).get_candidates() @@ -1019,6 +1023,7 @@ def test_node_assignment2(service_type, placement, hosts, spec=ServiceSpec(service_type, placement=placement), hosts=[HostSpec(h, labels=['foo']) for h in hosts], unreachable_hosts=[], + draining_hosts=[], daemons=daemons, ).place() assert len(hosts) == expected_len @@ -1053,6 +1058,7 @@ def test_node_assignment3(service_type, placement, hosts, spec=ServiceSpec(service_type, placement=placement), hosts=[HostSpec(h) for h in hosts], unreachable_hosts=[], + draining_hosts=[], daemons=daemons, ).place() assert len(hosts) == expected_len @@ -1150,6 +1156,7 @@ def test_node_assignment4(spec, networks, daemons, spec=spec, hosts=[HostSpec(h, labels=['foo']) for h in networks.keys()], unreachable_hosts=[], + draining_hosts=[], daemons=daemons, allow_colo=True, networks=networks, @@ -1236,6 +1243,7 @@ def test_bad_specs(service_type, placement, hosts, daemons, expected): spec=ServiceSpec(service_type, placement=placement), hosts=[HostSpec(h) for h in hosts], unreachable_hosts=[], + draining_hosts=[], daemons=daemons, ).place() assert str(e.value) == expected @@ -1412,6 +1420,7 @@ def test_active_assignment(service_type, placement, hosts, daemons, expected, ex spec=spec, hosts=[HostSpec(h) for h in hosts], unreachable_hosts=[], + draining_hosts=[], daemons=daemons, ).place() assert sorted([h.hostname for h in hosts]) in expected @@ -1509,6 +1518,7 @@ def test_unreachable_host(service_type, placement, hosts, unreachable_hosts, dae spec=spec, hosts=[HostSpec(h) for h in hosts], unreachable_hosts=[HostSpec(h) for h in unreachable_hosts], + draining_hosts=[], daemons=daemons, ).place() assert sorted([h.hostname for h in to_add]) in expected_add @@ -1585,6 +1595,7 @@ def test_remove_from_offline(service_type, placement, hosts, maintenance_hosts, spec=spec, hosts=host_specs, unreachable_hosts=[h for h in host_specs if h.status], + draining_hosts=[], daemons=daemons, ).place() assert sorted([h.hostname for h in to_add]) in expected_add diff --git a/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py b/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py index bec433fdcb7cc..41c0d96fcf3fe 100644 --- a/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py +++ b/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py @@ -34,6 +34,9 @@ class FakeCache: def get_unreachable_hosts(self): return self.unreachable_hosts + def get_draining_hosts(self): + return [] + @property def networks(self): return {h: {'a': {'b': ['c']}} for h in self.hosts} diff --git a/src/pybind/mgr/cephadm/tuned_profiles.py b/src/pybind/mgr/cephadm/tuned_profiles.py index f07f8f3e48fbc..7a5ce6fe60285 100644 --- a/src/pybind/mgr/cephadm/tuned_profiles.py +++ b/src/pybind/mgr/cephadm/tuned_profiles.py @@ -34,6 +34,7 @@ class TunedProfileUtils(): 'crash', placement=profile.placement), hosts=self.mgr.cache.get_schedulable_hosts(), unreachable_hosts=self.mgr.cache.get_unreachable_hosts(), + draining_hosts=self.mgr.cache.get_draining_hosts(), daemons=[], networks=self.mgr.cache.networks, )