mgr/cephadm: agent: kick serve loop if failed daemons change

If metadata from the agent causes the set of daemons marked as
failed to change we want to kick the serve loop immediately in order
to improve responsivess

Signed-off-by: Adam King <adking@redhat.com>

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 668de663fa1f426074d76bcef57c02e23bfc0ce6..993e96588f4bb0db003278427fb444d554d8f833 100644 (file)
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -12,6 +12,7 @@ from mgr_util import verify_tls_files
 from ceph.utils import datetime_now
 from ceph.deployment.inventory import Devices
 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec
+from orchestrator import DaemonDescriptionStatus
 
 from datetime import datetime, timedelta
 from OpenSSL import crypto
@@ -164,6 +165,9 @@ class HostData:
 
             # update timestamp of most recent agent update
             self.mgr.cache.agent_timestamp[host] = datetime_now()
+
+            error_daemons_old = set([dd.name() for dd in self.mgr.cache.get_error_daemons()])
+
             agents_down = []
             for h in self.mgr.cache.get_hosts():
                 if self.mgr.agent_helpers._agent_down(h):
@@ -193,6 +197,11 @@ class HostData:
                 ret = Devices.from_json(json.loads(data['volume']))
                 self.mgr.cache.update_host_devices(host, ret.devices)
 
+            if error_daemons_old != set([dd.name() for dd in self.mgr.cache.get_error_daemons()]):
+                self.mgr.log.info(
+                    f'Change detected in daemons in error state from {host} agent metadata. Kicking serve loop')
+                self.mgr._kick_serve_loop()
+
             if up_to_date:
                 was_out_of_date = not self.mgr.cache.all_host_metadata_up_to_date()
                 self.mgr.cache.metadata_up_to_date[host] = True
@@ -328,6 +337,8 @@ class CephadmAgentHelpers:
                 detail.append((f'Cephadm agent on host {agent} has not reported in '
                               f'{2.5 * self.mgr.agent_refresh_rate} seconds. Agent is assumed '
                                'down and host may be offline.'))
+            for dd in [d for d in self.mgr.cache.get_daemons_by_type('agent') if d.hostname in down_agent_hosts]:
+                dd.status = DaemonDescriptionStatus.error
             self.mgr.set_health_warning(
                 'CEPHADM_AGENT_DOWN',
                 summary='%d Cephadm Agent(s) are not reporting. Hosts may be offline' % (

diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 67ccc612ff78488cabe9e47b5dae9770e5cfdd9d..2336706ac589feb6ee813323ac50df75887bdd69 100644 (file)
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -817,6 +817,15 @@ class HostCache():
                 r.append(dd)
         return r
 
+    def get_error_daemons(self):
+        # type: () -> List[orchestrator.DaemonDescription]
+        r = []
+        for host, dm in self.daemons.items():
+            for name, dd in dm.items():
+                if dd.status is not None and dd.status == orchestrator.DaemonDescriptionStatus.error:
+                    r.append(dd)
+        return r
+
     def get_daemons_by_host(self, host: str) -> List[orchestrator.DaemonDescription]:
         return list(self.daemons.get(host, {}).values())
 

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 14f10af8095a76d4b15f0326198e337623c20377..138433f2b10e3d4b0db4973f8b5f6ec9d1463acf 100644 (file)
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -732,11 +732,10 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
     def update_failed_daemon_health_check(self) -> None:
         self.remove_health_warning('CEPHADM_FAILED_DAEMON')
         failed_daemons = []
-        for dd in self.cache.get_daemons():
-            if dd.status is not None and dd.status == DaemonDescriptionStatus.error:
-                failed_daemons.append('daemon %s on %s is in %s state' % (
-                    dd.name(), dd.hostname, dd.status_desc
-                ))
+        for dd in self.cache.get_error_daemons():
+            failed_daemons.append('daemon %s on %s is in %s state' % (
+                dd.name(), dd.hostname, dd.status_desc
+            ))
         if failed_daemons:
             self.set_health_warning('CEPHADM_FAILED_DAEMON', f'{len(failed_daemons)} failed cephadm daemon(s)', len(
                 failed_daemons), failed_daemons)