mgr/cephadm: suppress CEPHADM_FAILED_DAEMON for services being removed

When a service is deleted, a daemon belonging to it may transiently
enter a failed state before orphan cleanup removes it. This happens
when there is a race between dependency-changed REDEPLOY and a
service removal going on paralelly.

Skip daemons in update_failed_daemon_health_check whose service is
already in spec_deleted.

Fixes: https://tracker.ceph.com/issues/77944
Signed-off-by: Avan Thakkar <athakkar@redhat.com>

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 24c6483007ad478af96cb334e0aad0237093ec94..12c45526d2f4ca2ab067657a8b5d5e97c4a69618 100644 (file)
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1184,10 +1184,19 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule):
     def update_failed_daemon_health_check(self) -> None:
         failed_daemons = []
         for dd in self.cache.get_error_daemons():
-            if dd.daemon_type != 'agent':  # agents tracked by CEPHADM_AGENT_DOWN
-                failed_daemons.append('daemon %s on %s is in %s state' % (
-                    dd.name(), dd.hostname, dd.status_desc
-                ))
+            if dd.daemon_type == 'agent':  # agents tracked by CEPHADM_AGENT_DOWN
+                continue
+            assert dd.daemon_type is not None
+            svc_type = daemon_type_to_service(dd.daemon_type)
+            if (svc_type in ServiceSpec.REQUIRES_SERVICE_ID
+                    and dd.service_name() in self.spec_store.spec_deleted):
+                # Service is being removed; daemon failure will be
+                # resolved by orphan cleanup. Don't raise a health
+                # alert that clears itself once removal completes.
+                continue
+            failed_daemons.append('daemon %s on %s is in %s state' % (
+                dd.name(), dd.hostname, dd.status_desc
+            ))
         self.remove_health_warning('CEPHADM_FAILED_DAEMON')
         if failed_daemons:
             self.set_health_warning('CEPHADM_FAILED_DAEMON', f'{len(failed_daemons)} failed cephadm daemon(s)', len(