mgr/cephadm: adjust deployment logic to allow multiple daemons per host

If we are told to deploy multiple instances on a host, do it.

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit 7c083af2d8cd79518be279b37722297ca9c768c3)

diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py
index f63a1a2029ed1813e24821a273477570486716c1..b6cb3f4841bc25673ca7dafb47f23c0cc6ce0109 100644 (file)
--- a/src/pybind/mgr/cephadm/schedule.py
+++ b/src/pybind/mgr/cephadm/schedule.py
@@ -194,12 +194,12 @@ class HostAssignment(object):
         # ask the scheduler to return a set of hosts with a up to the value of <count>
         return self.scheduler.place(hosts, count)
 
-    def add_daemon_hosts(self, host_pool: List[HostPlacementSpec]) -> Set[HostPlacementSpec]:
+    def add_daemon_hosts(self, host_pool: List[HostPlacementSpec]) -> List[HostPlacementSpec]:
         hosts_with_daemons = {d.hostname for d in self.daemons}
-        _add_daemon_hosts = set()
+        _add_daemon_hosts = []  # type: List[HostPlacementSpec]
         for host in host_pool:
             if host.hostname not in hosts_with_daemons:
-                _add_daemon_hosts.add(host)
+                _add_daemon_hosts.append(host)
         return _add_daemon_hosts
 
     def remove_daemon_hosts(self, host_pool: List[HostPlacementSpec]) -> Set[DaemonDescription]:
@@ -208,6 +208,8 @@ class HostAssignment(object):
         for d in self.daemons:
             if d.hostname not in target_hosts:
                 _remove_daemon_hosts.add(d)
+            else:
+                target_hosts.remove(d.hostname)
         return _remove_daemon_hosts
 
     def get_candidates(self) -> List[HostPlacementSpec]:
@@ -248,7 +250,10 @@ class HostAssignment(object):
         return existing
 
 
-def merge_hostspecs(lh: List[HostPlacementSpec], rh: List[HostPlacementSpec]) -> Iterable[HostPlacementSpec]:
+def merge_hostspecs(
+        lh: List[HostPlacementSpec],
+        rh: List[HostPlacementSpec]
+) -> Iterable[HostPlacementSpec]:
     """
     Merge two lists of HostPlacementSpec by hostname. always returns `lh` first.
 
@@ -262,7 +267,10 @@ def merge_hostspecs(lh: List[HostPlacementSpec], rh: List[HostPlacementSpec]) ->
     yield from (h for h in rh if h.hostname not in lh_names)
 
 
-def difference_hostspecs(lh: List[HostPlacementSpec], rh: List[HostPlacementSpec]) -> List[HostPlacementSpec]:
+def difference_hostspecs(
+        lh: List[HostPlacementSpec],
+        rh: List[HostPlacementSpec]
+) -> List[HostPlacementSpec]:
     """
     returns lh "minus" rh by hostname.
 

diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index e2c2fa9a72749185bd8d9c05d4a42ffc40929617..f2d1f1254a7f67ad8fd8108842c13613628b6fd0 100644 (file)
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -574,11 +574,11 @@ class CephadmServe:
         # add any?
         did_config = False
 
-        add_daemon_hosts: Set[HostPlacementSpec] = ha.add_daemon_hosts(hosts)
+        add_daemon_hosts: List[HostPlacementSpec] = ha.add_daemon_hosts(hosts)
         self.log.debug('Hosts that will receive new daemons: %s' % add_daemon_hosts)
 
-        remove_daemon_hosts: Set[orchestrator.DaemonDescription] = ha.remove_daemon_hosts(hosts)
-        self.log.debug('Hosts that will loose daemons: %s' % remove_daemon_hosts)
+        remove_daemons: Set[orchestrator.DaemonDescription] = ha.remove_daemon_hosts(hosts)
+        self.log.debug('Daemons that will be removed: %s' % remove_daemons)
 
         if service_type == 'ha-rgw':
             spec = self.update_ha_rgw_definitive_hosts(spec, hosts, add_daemon_hosts)
@@ -622,18 +622,18 @@ class CephadmServe:
                 daemons.append(sd)
 
         # remove any?
-        def _ok_to_stop(remove_daemon_hosts: Set[orchestrator.DaemonDescription]) -> bool:
-            daemon_ids = [d.daemon_id for d in remove_daemon_hosts]
+        def _ok_to_stop(remove_daemons: Set[orchestrator.DaemonDescription]) -> bool:
+            daemon_ids = [d.daemon_id for d in remove_daemons]
             assert None not in daemon_ids
-            # setting force flag retains previous behavior, should revisit later.
+            # setting force flag retains previous behavior
             r = self.mgr.cephadm_services[service_type].ok_to_stop(
                 cast(List[str], daemon_ids), force=True)
             return not r.retval
 
-        while remove_daemon_hosts and not _ok_to_stop(remove_daemon_hosts):
+        while remove_daemons and not _ok_to_stop(remove_daemons):
             # let's find a subset that is ok-to-stop
-            remove_daemon_hosts.pop()
-        for d in remove_daemon_hosts:
+            remove_daemons.pop()
+        for d in remove_daemons:
             r = True
             # NOTE: we are passing the 'force' flag here, which means
             # we can delete a mon instances data.
@@ -768,8 +768,12 @@ class CephadmServe:
     # if definitive host list has changed, all ha-rgw daemons must get new
     # config, including those that are already on the correct host and not
     # going to be deployed
-    def update_ha_rgw_definitive_hosts(self, spec: ServiceSpec, hosts: List[HostPlacementSpec],
-                                       add_hosts: Set[HostPlacementSpec]) -> HA_RGWSpec:
+    def update_ha_rgw_definitive_hosts(
+            self,
+            spec: ServiceSpec,
+            hosts: List[HostPlacementSpec],
+            add_hosts: List[HostPlacementSpec]
+    ) -> HA_RGWSpec:
         spec = cast(HA_RGWSpec, spec)
         if not (set(hosts) == set(spec.definitive_host_list)):
             spec.definitive_host_list = hosts