From: Adam King <adking@redhat.com>
Date: Thu, 21 Oct 2021 12:36:33 +0000 (-0400)
Subject: mgr/cephadm: agent: kick serve loop if failed daemons change
X-Git-Tag: v17.1.0~494^2~10
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f18432fffd3cddcacface0e8636327af90f41ec1;p=ceph.git

mgr/cephadm: agent: kick serve loop if failed daemons change

If metadata from the agent causes the set of daemons marked as
failed to change we want to kick the serve loop immediately in order
to improve responsivess

Signed-off-by: Adam King <adking@redhat.com>
---

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 668de663fa1..993e96588f4 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -12,6 +12,7 @@ from mgr_util import verify_tls_files
 from ceph.utils import datetime_now
 from ceph.deployment.inventory import Devices
 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec
+from orchestrator import DaemonDescriptionStatus
 
 from datetime import datetime, timedelta
 from OpenSSL import crypto
@@ -164,6 +165,9 @@ class HostData:
 
             # update timestamp of most recent agent update
             self.mgr.cache.agent_timestamp[host] = datetime_now()
+
+            error_daemons_old = set([dd.name() for dd in self.mgr.cache.get_error_daemons()])
+
             agents_down = []
             for h in self.mgr.cache.get_hosts():
                 if self.mgr.agent_helpers._agent_down(h):
@@ -193,6 +197,11 @@ class HostData:
                 ret = Devices.from_json(json.loads(data['volume']))
                 self.mgr.cache.update_host_devices(host, ret.devices)
 
+            if error_daemons_old != set([dd.name() for dd in self.mgr.cache.get_error_daemons()]):
+                self.mgr.log.info(
+                    f'Change detected in daemons in error state from {host} agent metadata. Kicking serve loop')
+                self.mgr._kick_serve_loop()
+
             if up_to_date:
                 was_out_of_date = not self.mgr.cache.all_host_metadata_up_to_date()
                 self.mgr.cache.metadata_up_to_date[host] = True
@@ -328,6 +337,8 @@ class CephadmAgentHelpers:
                 detail.append((f'Cephadm agent on host {agent} has not reported in '
                               f'{2.5 * self.mgr.agent_refresh_rate} seconds. Agent is assumed '
                                'down and host may be offline.'))
+            for dd in [d for d in self.mgr.cache.get_daemons_by_type('agent') if d.hostname in down_agent_hosts]:
+                dd.status = DaemonDescriptionStatus.error
             self.mgr.set_health_warning(
                 'CEPHADM_AGENT_DOWN',
                 summary='%d Cephadm Agent(s) are not reporting. Hosts may be offline' % (
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 67ccc612ff7..2336706ac58 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -817,6 +817,15 @@ class HostCache():
                 r.append(dd)
         return r
 
+    def get_error_daemons(self):
+        # type: () -> List[orchestrator.DaemonDescription]
+        r = []
+        for host, dm in self.daemons.items():
+            for name, dd in dm.items():
+                if dd.status is not None and dd.status == orchestrator.DaemonDescriptionStatus.error:
+                    r.append(dd)
+        return r
+
     def get_daemons_by_host(self, host: str) -> List[orchestrator.DaemonDescription]:
         return list(self.daemons.get(host, {}).values())
 
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 14f10af8095..138433f2b10 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -732,11 +732,10 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
     def update_failed_daemon_health_check(self) -> None:
         self.remove_health_warning('CEPHADM_FAILED_DAEMON')
         failed_daemons = []
-        for dd in self.cache.get_daemons():
-            if dd.status is not None and dd.status == DaemonDescriptionStatus.error:
-                failed_daemons.append('daemon %s on %s is in %s state' % (
-                    dd.name(), dd.hostname, dd.status_desc
-                ))
+        for dd in self.cache.get_error_daemons():
+            failed_daemons.append('daemon %s on %s is in %s state' % (
+                dd.name(), dd.hostname, dd.status_desc
+            ))
         if failed_daemons:
             self.set_health_warning('CEPHADM_FAILED_DAEMON', f'{len(failed_daemons)} failed cephadm daemon(s)', len(
                 failed_daemons), failed_daemons)