From e545ed90e9bf863b59723200172a833dfe69ae7b Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 9 Mar 2021 18:49:51 -0500 Subject: [PATCH] mgr/cephadm/schedule: allow colocation of certain daemon types For certain daemon types, we can deploy more than one per host (mds, rbd-mirror, rgw). Signed-off-by: Sage Weil (cherry picked from commit 6d0098d1cd6b639880e12a5f471dd8d31e7c0017) --- src/pybind/mgr/cephadm/module.py | 2 + src/pybind/mgr/cephadm/schedule.py | 111 ++++++++---------- .../mgr/cephadm/tests/test_scheduling.py | 39 +++++- src/pybind/mgr/orchestrator/_interface.py | 10 +- 4 files changed, 96 insertions(+), 66 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index ca110de5f86..0adaf5dd5b6 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2087,6 +2087,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, spec=spec, hosts=self._hosts_with_daemon_inventory(), daemons=self.cache.get_daemons_by_service(spec.service_name()), + allow_colo=self.cephadm_services[spec.service_type].allow_colo(), ) ha.validate() hosts = ha.place() @@ -2145,6 +2146,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, spec=spec, hosts=self.inventory.all_specs(), # All hosts, even those without daemon refresh daemons=self.cache.get_daemons_by_service(spec.service_name()), + allow_colo=self.cephadm_services[spec.service_type].allow_colo(), ).validate() self.log.info('Saving service %s spec with placement %s' % ( diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py index 785a1bebaf8..31ce9b9a626 100644 --- a/src/pybind/mgr/cephadm/schedule.py +++ b/src/pybind/mgr/cephadm/schedule.py @@ -125,6 +125,8 @@ class HostAssignment(object): # get candidate hosts based on [hosts, label, host_pattern] candidates = self.get_candidates() # type: List[HostPlacementSpec] + if not candidates: + return [] # sigh # If we don't have the list of candidates is definitive. if count is None: @@ -135,55 +137,51 @@ class HostAssignment(object): per_host = 1 return candidates * per_host - # prefer hosts that already have services. - # this avoids re-assigning to _new_ hosts - # and constant re-distribution of hosts when new nodes are - # added to the cluster - hosts_with_daemons = self.hosts_with_daemons(candidates) - - # The amount of hosts that need to be selected in order to fulfill count. - need = count - len(hosts_with_daemons) - - # hostspecs that do not have daemons on them but are still candidates. - others = difference_hostspecs(candidates, hosts_with_daemons) - - # we don't need any additional hosts - if need < 0: - final_candidates = self.prefer_hosts_with_active_daemons(hosts_with_daemons, count) - else: - # ask the scheduler to return a set of hosts with a up to the value of - others = self.scheduler.place(others, need) - logger.debug('Combine hosts with existing daemons %s + new hosts %s' % ( - hosts_with_daemons, others)) - # if a host already has the anticipated daemon, merge it with the candidates - # to get a list of HostPlacementSpec that can be deployed on. - final_candidates = list(merge_hostspecs(hosts_with_daemons, others)) - - return final_candidates - - def get_hosts_with_active_daemon(self, hosts: List[HostPlacementSpec]) -> List[HostPlacementSpec]: - active_hosts: List['HostPlacementSpec'] = [] - for daemon in self.daemons: - if daemon.is_active: - for h in hosts: - if h.hostname == daemon.hostname: - active_hosts.append(h) - # remove duplicates before returning - return list(dict.fromkeys(active_hosts)) - - def prefer_hosts_with_active_daemons(self, hosts: List[HostPlacementSpec], count: int) -> List[HostPlacementSpec]: - # try to prefer host with active daemon if possible - active_hosts = self.get_hosts_with_active_daemon(hosts) - if len(active_hosts) != 0 and count > 0: - for host in active_hosts: - hosts.remove(host) - if len(active_hosts) >= count: - return self.scheduler.place(active_hosts, count) - else: - return list(merge_hostspecs(self.scheduler.place(active_hosts, count), - self.scheduler.place(hosts, count - len(active_hosts)))) - # ask the scheduler to return a set of hosts with a up to the value of - return self.scheduler.place(hosts, count) + if self.allow_colo: + # consider enough slots to fit target count + per_host = 1 + ((count - 1) // len(candidates)) + candidates = candidates * per_host + + # consider active daemons first + daemons = [ + d for d in self.daemons if d.is_active + ] + [ + d for d in self.daemons if not d.is_active + ] + + # sort candidates into existing/used slots that already have a + # daemon, and others (the rest) + existing_active: List[HostPlacementSpec] = [] + existing_standby: List[HostPlacementSpec] = [] + existing: List[HostPlacementSpec] = [] + others = candidates + for d in daemons: + hs = d.get_host_placement() + for i in others: + if i == hs: + others.remove(i) + if d.is_active: + existing_active.append(hs) + else: + existing_standby.append(hs) + existing.append(hs) + break + logger.debug('Hosts with existing active daemons: {}'.format(existing_active)) + logger.debug('Hosts with existing standby daemons: {}'.format(existing_standby)) + + # The number of new slots that need to be selected in order to fulfill count + need = count - len(existing) + + # we don't need any additional placements + if need <= 0: + del existing[count:] + return existing + + # ask the scheduler to select additional slots + chosen = self.scheduler.place(others, need) + logger.debug('Combine hosts with existing daemons %s + new hosts %s' % ( + existing, chosen)) + return existing + chosen def add_daemon_hosts(self, host_pool: List[HostPlacementSpec]) -> List[HostPlacementSpec]: hosts_with_daemons = {d.hostname for d in self.daemons} @@ -239,21 +237,8 @@ class HostAssignment(object): # gen seed off of self.spec to make shuffling deterministic seed = hash(self.spec.service_name()) random.Random(seed).shuffle(hosts) - return hosts - def hosts_with_daemons(self, candidates: List[HostPlacementSpec]) -> List[HostPlacementSpec]: - """ - Prefer hosts with daemons. Otherwise we'll constantly schedule daemons - on different hosts all the time. This is about keeping daemons where - they are. This isn't about co-locating. - """ - hosts_with_daemons = {d.hostname for d in self.daemons} - - # calc existing daemons (that aren't already in chosen) - existing = [hs for hs in candidates if hs.hostname in hosts_with_daemons] - - logger.debug('Hosts with existing daemons: {}'.format(existing)) - return existing + return hosts def merge_hostspecs( diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py index 2b370588bd5..990e84cc076 100644 --- a/src/pybind/mgr/cephadm/tests/test_scheduling.py +++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py @@ -363,14 +363,22 @@ class NodeAssignmentTest(NamedTuple): ['host1', 'host2', 'host3', 'host1', 'host2', 'host3'] ), # count that is bigger than the amount of hosts. Truncate to len(hosts) - # RGWs should not be co-located to each other. + # mgr should not be co-located to each other. NodeAssignmentTest( - 'rgw', + 'mgr', PlacementSpec(count=4), 'host1 host2 host3'.split(), [], ['host1', 'host2', 'host3'] ), + # count that is bigger than the amount of hosts; wrap around. + NodeAssignmentTest( + 'mds', + PlacementSpec(count=6), + 'host1 host2 host3'.split(), + [], + ['host1', 'host2', 'host3', 'host1', 'host2', 'host3'] + ), # count + partial host list NodeAssignmentTest( 'mgr', @@ -382,6 +390,17 @@ class NodeAssignmentTest(NamedTuple): ], ['host3'] ), + # count + partial host list (with colo) + NodeAssignmentTest( + 'mds', + PlacementSpec(count=3, hosts=['host3']), + 'host1 host2 host3'.split(), + [ + DaemonDescription('mgr', 'a', 'host1'), + DaemonDescription('mgr', 'b', 'host2'), + ], + ['host3', 'host3', 'host3'] + ), # count 1 + partial host list NodeAssignmentTest( 'mgr', @@ -431,6 +450,22 @@ class NodeAssignmentTest(NamedTuple): [], ['host1', 'host2', 'host3'] ), + # label + count (truncate to host list) + NodeAssignmentTest( + 'mgr', + PlacementSpec(count=4, label='foo'), + 'host1 host2 host3'.split(), + [], + ['host1', 'host2', 'host3'] + ), + # label + count (with colo) + NodeAssignmentTest( + 'mds', + PlacementSpec(count=6, label='foo'), + 'host1 host2 host3'.split(), + [], + ['host1', 'host2', 'host3', 'host1', 'host2', 'host3'] + ), # label only + count_per_hst NodeAssignmentTest( 'mds', diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index f44a5983e95..29b8603818e 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -31,7 +31,7 @@ import yaml from ceph.deployment import inventory from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, \ - ServiceSpecValidationError, IscsiServiceSpec, HA_RGWSpec + ServiceSpecValidationError, IscsiServiceSpec, HA_RGWSpec, HostPlacementSpec from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.hostspec import HostSpec from ceph.utils import datetime_to_str, str_to_datetime @@ -920,6 +920,14 @@ class DaemonDescription(object): return f'{daemon_type_to_service(self.daemon_type)}.{self.service_id()}' return daemon_type_to_service(self.daemon_type) + def get_host_placement(self) -> HostPlacementSpec: + return HostPlacementSpec( + hostname=self.hostname or '', + # FIXME: include the ip:port here? + network='', + name='', + ) + def __repr__(self) -> str: return "({type}.{id})".format(type=self.daemon_type, id=self.daemon_id) -- 2.47.3