cephadm: report intentionally stopped daemons as stopped

node-exporter exits with status 143 when terminated via SIGTERM.
Systemd treats this as a failure by default, causing cephadm to
report an intentionally stopped daemon as being in an error state.

Allow daemon forms to provide daemon-specific success exit status
codes and generate a corresponding systemd SuccessExitStatus
drop-in when required.

For node-exporter, configure exit status 143 as successful so
intentional stops are reported as stopped rather than failed.

Fixes: https://tracker.ceph.com/issues/68863
Signed-off-by: Shubha Jain <SHUBHA.JAIN1@ibm.com>

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 7cf65220c803de9880d54d0a00fbd0602e813c55..72e3c77e75ed022c83095013f604b038554714cd 100755 (executable)
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1312,7 +1312,11 @@ def deploy_daemon_units(
         DaemonSubIdentity.must(sc.identity) for sc in sidecars or []
     ]
     systemd_unit.update_files(
-        ctx, ident, init_container_ids=ic_ids, sidecar_ids=sc_ids
+        ctx,
+        ident,
+        init_container_ids=ic_ids,
+        sidecar_ids=sc_ids,
+        success_exit_status=container.success_exit_status,
     )
     call_throws(ctx, ['systemctl', 'daemon-reload'])
 

diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index c38a27d8daab63a1e26a33de76e5deb68a825379..6c18ffdb4c16a781fb866bee68ab7aecdce03cfd 100644 (file)
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -254,6 +254,7 @@ class CephContainer(BasicContainer):
         self.remove = True
         self.ipc = 'host'
         self.network = 'host' if self.host_network else ''
+        self.success_exit_status: Optional[List[int]] = None
 
     @classmethod
     def for_daemon(

diff --git a/src/cephadm/cephadmlib/daemons/monitoring.py b/src/cephadm/cephadmlib/daemons/monitoring.py
index d93fc6c68ecc4cf2bbbb6551b26f1cb26939f1b1..a992ca9db169862c4f5369b0841ba673e4e09cdd 100644 (file)
--- a/src/cephadm/cephadmlib/daemons/monitoring.py
+++ b/src/cephadm/cephadmlib/daemons/monitoring.py
@@ -215,7 +215,10 @@ class Monitoring(ContainerDaemonForm):
     def container(self, ctx: CephadmContext) -> CephContainer:
         self._prevalidate(ctx)
         ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
+        ctr = to_deployment_container(ctx, ctr)
+        if self.identity.daemon_type == 'node-exporter':
+            ctr.success_exit_status = [143]
+        return ctr
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return self.extract_uid_gid(ctx, self.identity.daemon_type)

diff --git a/src/cephadm/cephadmlib/systemd_unit.py b/src/cephadm/cephadmlib/systemd_unit.py
index d3543174a8df39f7d4d4d77affaa2e10e8d389d1..c76fee4e74795b18e5be62973b8a8783f63c2d26 100644 (file)
--- a/src/cephadm/cephadmlib/systemd_unit.py
+++ b/src/cephadm/cephadmlib/systemd_unit.py
@@ -16,6 +16,7 @@ from .logging import write_cluster_logrotate_config
 
 
 _DROP_IN_FILENAME = '99-cephadm.conf'
+_SUCCESS_EXIT_STATUS_DROP_IN_FMT = '90-cephadm-{daemon_type}.conf'
 
 
 class PathInfo:
@@ -100,6 +101,39 @@ def _write_sidecar_unit_file(
     )
 
 
+def _install_success_exit_status_drop_in(
+    pinfo: PathInfo,
+    ident: DaemonIdentity,
+    exit_codes: List[int],
+) -> None:
+    """Install a drop-in that marks extra exit codes as success.
+
+    Container runtimes exit with 143 (128 + SIGTERM) on a clean stop.
+    Without an explicit SuccessExitStatus directive systemd treats this
+    as a failure, causing cephadm to report the daemon as ``error``
+    instead of ``stopped``.
+
+    The list of exit codes is carried on the ``CephContainer`` object
+    and populated by the daemon's ``container()`` method, keeping
+    daemon-specific knowledge out of this module.
+    """
+    if not exit_codes:
+        return
+    pinfo.drop_in_file.parent.mkdir(parents=True, exist_ok=True)
+    fname = _SUCCESS_EXIT_STATUS_DROP_IN_FMT.format(
+        daemon_type=ident.daemon_type
+    )
+    dest = pinfo.drop_in_file.parent / fname
+    codes = ' '.join(str(c) for c in exit_codes)
+    content = (
+        '# generated by cephadm\n'
+        '[Service]\n'
+        f'SuccessExitStatus={codes}\n'
+    )
+    with write_new(dest, perms=None) as f:
+        f.write(content)
+
+
 def _install_extended_systemd_services(
     ctx: CephadmContext,
     pinfo: PathInfo,
@@ -203,6 +237,7 @@ def update_files(
     *,
     init_container_ids: Optional[List[DaemonSubIdentity]] = None,
     sidecar_ids: Optional[List[DaemonSubIdentity]] = None,
+    success_exit_status: Optional[List[int]] = None,
 ) -> None:
     _install_base_units(ctx, ident.fsid)
     unit = _get_unit_file(ctx, ident.fsid)
@@ -212,6 +247,9 @@ def update_files(
     _install_extended_systemd_services(
         ctx, pathinfo, ident, bool(init_container_ids)
     )
+    _install_success_exit_status_drop_in(
+        pathinfo, ident, success_exit_status or []
+    )
 
 
 def sidecars_from_dropin(

diff --git a/src/cephadm/tests/test_unit_file.py b/src/cephadm/tests/test_unit_file.py
index 74cd89c1a82338d08a3585548eebdf5640070480..43c2bd2465354efb9cb7f48851166b485f4b262d 100644 (file)
--- a/src/cephadm/tests/test_unit_file.py
+++ b/src/cephadm/tests/test_unit_file.py
@@ -19,6 +19,7 @@ from tests.fixtures import (
 from cephadmlib import context
 from cephadmlib import systemd_unit
 from cephadmlib.constants import CGROUPS_SPLIT_PODMAN_VERSION
+from cephadmlib.daemon_identity import DaemonIdentity
 
 _cephadm = import_cephadm()
 
@@ -147,3 +148,87 @@ def test_new_podman():
         '[Install]',
         'WantedBy=ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9.target',
     ]
+
+
+_FSID = '9b9d7609-f4d5-4aba-94c8-effa764d96c9'
+
+
+def test_success_exit_status_drop_in_written(tmp_path):
+    """When exit codes are provided, a drop-in with SuccessExitStatus is
+    written into the service's .d/ directory."""
+    ident = DaemonIdentity(_FSID, 'node-exporter', 'ceph-node-00')
+    pinfo = systemd_unit.PathInfo(tmp_path, ident)
+    systemd_unit._install_success_exit_status_drop_in(
+        pinfo, ident, [143]
+    )
+    fname = systemd_unit._SUCCESS_EXIT_STATUS_DROP_IN_FMT.format(
+        daemon_type='node-exporter'
+    )
+    drop_in = pinfo.drop_in_file.parent / fname
+    assert drop_in.exists()
+    assert fname == '90-cephadm-node-exporter.conf'
+    content = drop_in.read_text()
+    assert '[Service]' in content
+    assert 'SuccessExitStatus=143' in content
+
+
+def test_success_exit_status_drop_in_not_written_for_empty_list(tmp_path):
+    """When exit codes list is empty, no drop-in is created."""
+    ident = DaemonIdentity(_FSID, 'mgr', 'host1')
+    pinfo = systemd_unit.PathInfo(tmp_path, ident)
+    systemd_unit._install_success_exit_status_drop_in(
+        pinfo, ident, []
+    )
+    fname = systemd_unit._SUCCESS_EXIT_STATUS_DROP_IN_FMT.format(
+        daemon_type='mgr'
+    )
+    drop_in = pinfo.drop_in_file.parent / fname
+    assert not drop_in.exists()
+
+
+@mock.patch('cephadmlib.daemons.monitoring.Monitoring._prevalidate')
+@mock.patch('cephadmlib.daemons.monitoring.daemon_to_container')
+@mock.patch('cephadmlib.daemons.monitoring.to_deployment_container')
+def test_monitoring_node_exporter_container_has_success_exit_status(
+    _to_deploy, _to_ctr, _preval
+):
+    """Monitoring.container() sets success_exit_status=[143] for
+    node-exporter."""
+    from cephadmlib.daemons.monitoring import Monitoring
+    from cephadmlib.container_types import CephContainer
+
+    ctx = context.CephadmContext()
+    ctx.container_engine = mock_podman()
+    fake_ctr = mock.MagicMock(spec=CephContainer)
+    fake_ctr.success_exit_status = None
+    _to_deploy.return_value = fake_ctr
+    ident = DaemonIdentity(_FSID, 'node-exporter', 'host1')
+    daemon = Monitoring(ctx, ident)
+    ctr = daemon.container(ctx)
+    assert ctr.success_exit_status == [143]
+
+
+@mock.patch('cephadmlib.daemons.monitoring.Monitoring._prevalidate')
+@mock.patch('cephadmlib.daemons.monitoring.daemon_to_container')
+@mock.patch('cephadmlib.daemons.monitoring.to_deployment_container')
+@pytest.mark.parametrize(
+    'daemon_type',
+    ['prometheus', 'grafana', 'alertmanager', 'loki', 'promtail'],
+)
+def test_monitoring_other_types_no_success_exit_status(
+    _to_deploy, _to_ctr, _preval, daemon_type
+):
+    """Monitoring.container() leaves success_exit_status as None for
+    non-node-exporter types."""
+    from cephadmlib.daemons.monitoring import Monitoring
+    from cephadmlib.container_types import CephContainer
+
+    ctx = context.CephadmContext()
+    ctx.container_engine = mock_podman()
+    fake_ctr = mock.MagicMock(spec=CephContainer)
+    fake_ctr.success_exit_status = None
+    _to_deploy.return_value = fake_ctr
+    ident = DaemonIdentity(_FSID, daemon_type, 'host1')
+    daemon = Monitoring(ctx, ident)
+    ctr = daemon.container(ctx)
+    assert ctr.success_exit_status is None