cephadm: start disabled services on host-maintenance exit

NFS and keepalived are deployed without systemd enable (DISABLED_SERVICES).
When a host exits maintenance, re-enabling ceph-<fsid>.target only starts
enabled units, so these daemons remain stopped.

Fixes: https://tracker.ceph.com/issues/77954
Signed-off-by: Shweta Bhosale <Shweta.Bhosale1@ibm.com>

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index f14fe91b85ac1750c7afe7a5b448c6ec31b26f74..e32dd9fe6e5b5c3fd6e32c3024b2239f6387740c 100755 (executable)
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -60,6 +60,7 @@ from cephadmlib.constants import (
     SYSCTL_DIR,
     UNIT_DIR,
     DAEMON_FAILED_ERROR,
+    DISABLED_SERVICES,
 )
 from cephadmlib.context import CephadmContext
 from cephadmlib.context_getters import (
@@ -138,7 +139,13 @@ from cephadmlib.logging import (
     Highlight,
     LogDestination,
 )
-from cephadmlib.systemd import check_unit, check_units, terminate_service, enable_service
+from cephadmlib.systemd import (
+    check_unit,
+    check_units,
+    terminate_service,
+    enable_service,
+    start_disabled_services_after_maintenance_exit,
+)
 from cephadmlib import systemd_unit
 from cephadmlib.signals import send_signal_to_container_entrypoint
 from cephadmlib import runscripts
@@ -1177,7 +1184,7 @@ def deploy_daemon(
             if c:
                 # Disable automatic systemd enable for NFS and keepalived; the mgr
                 # starts them when appropriate (see cephadm serve / DISABLED_SERVICES).
-                enable_daemon = daemon_type not in ('nfs', 'keepalived')
+                enable_daemon = daemon_type not in DISABLED_SERVICES
                 deploy_daemon_units(
                     ctx,
                     ident,
@@ -4803,26 +4810,28 @@ def change_maintenance_mode(ctx: CephadmContext) -> str:
         # return success here or host will be permanently stuck in maintenance mode
         # as no daemons can be deployed so no systemd target will ever exist to disable.
         if not target_exists(ctx):
-            return 'skipped - systemd target not present on this host. Host removed from maintenance mode.'
-        # exit maintenance request
-        if not systemd_target_state(ctx, target):
+            msg = (
+                'skipped - systemd target not present on this host. '
+                'Host removed from maintenance mode.')
+        elif not systemd_target_state(ctx, target):
             _out, _err, code = call(ctx,
                                     ['systemctl', 'enable', target],
                                     verbosity=CallVerbosity.DEBUG)
             if code:
                 logger.error(f'Failed to enable the {target} target')
                 return 'failed - unable to enable the target'
-            else:
-                # starting a target waits by default
-                _out, _err, code = call(ctx,
-                                        ['systemctl', 'start', target],
-                                        verbosity=CallVerbosity.DEBUG)
-                if code:
-                    logger.error(f'Failed to start the {target} target')
-                    return 'failed - unable to start the target'
-                else:
-                    return f'success - systemd target {target} enabled and started'
-        return f'success - systemd target {target} enabled and started'
+            _out, _err, code = call(ctx,
+                                    ['systemctl', 'start', target],
+                                    verbosity=CallVerbosity.DEBUG)
+            if code:
+                logger.error(f'Failed to start the {target} target')
+                return 'failed - unable to start the target'
+            msg = f'success - systemd target {target} enabled and started'
+        else:
+            msg = f'success - systemd target {target} enabled and started'
+
+        start_disabled_services_after_maintenance_exit(ctx)
+        return msg
 
 
 @infer_fsid

diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index 874c28742e3be1a207a3b2e3c94de49ceab8db7f..79a9c61aba408ec013cee27ebb2b10619ced2e74 100644 (file)
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -38,5 +38,8 @@ UID_NOBODY = 65534
 GID_NOGROUP = 65534
 DAEMON_FAILED_ERROR = 17
 
+# Daemons not systemd-enabled on deploy; mgr starts them when appropriate.
+DISABLED_SERVICES = ['nfs', 'keepalived']
+
 # Host labels
 ADMIN_LABEL = '_admin'

diff --git a/src/cephadm/cephadmlib/systemd.py b/src/cephadm/cephadmlib/systemd.py
index 1d2668a480383f86a42a3cf442905fafc4737e23..37c2cf979fb73c9a791280a4679f1ace325ee305 100644 (file)
--- a/src/cephadm/cephadmlib/systemd.py
+++ b/src/cephadm/cephadmlib/systemd.py
@@ -4,8 +4,10 @@ import logging
 
 from typing import Tuple, List
 
+from .constants import DISABLED_SERVICES
 from .context import CephadmContext
 from .call_wrappers import call, CallVerbosity
+from .listing import DaemonEntry, daemons_matching
 
 logger = logging.getLogger()
 
@@ -93,3 +95,32 @@ def enable_service(ctx: CephadmContext, service_name: str) -> None:
         ['systemctl', 'enable', '--now', service_name],
         verbosity=CallVerbosity.DEBUG,
     )
+
+
+def start_disabled_services_after_maintenance_exit(
+    ctx: CephadmContext,
+) -> None:
+    """Start nfs/keepalived units after host-maintenance exit."""
+    if not ctx.fsid:
+        return
+    for daemon_type in DISABLED_SERVICES:
+        for entry in daemons_matching(
+            ctx, fsid=ctx.fsid, daemon_type=daemon_type
+        ):
+            if isinstance(entry, DaemonEntry):
+                unit = entry.identity.unit_name
+            else:
+                unit = entry.status['systemd_unit']
+            _out, _err, code = call(
+                ctx,
+                ['systemctl', 'start', unit],
+                verbosity=CallVerbosity.DEBUG,
+            )
+            if code:
+                logger.warning(
+                    'Failed to start %s after maintenance exit: %s',
+                    unit,
+                    _err,
+                )
+            else:
+                logger.info('Started %s after maintenance exit', unit)

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 43ba03dd771aeba590dde8c44cad9e4dd7c6c2b5..2682ca9930ef0eaa12caf4a1d4e9727a8ebeac72 100644 (file)
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1735,6 +1735,74 @@ class TestMaintenance:
         assert retval.startswith('failed')
 
 
+class TestMaintenanceExitDisabledServices(object):
+    fsid = '00000000-0000-0000-0000-000000000001'
+
+    @staticmethod
+    def _nfs_daemon_entry(fsid: str):
+        from cephadmlib.daemon_identity import DaemonIdentity
+        from cephadmlib.listing import DaemonEntry
+
+        identity = DaemonIdentity.from_name(fsid, 'nfs.foo.host1')
+        return DaemonEntry(
+            identity=identity,
+            status={
+                'style': 'cephadm:v1',
+                'name': 'nfs.foo.host1',
+                'fsid': fsid,
+                'systemd_unit': identity.unit_name,
+            },
+            data_dir='/var/lib/ceph',
+        )
+
+    @mock.patch('cephadmlib.systemd.call')
+    def test_start_disabled_services_after_maintenance_exit(self, _call):
+        from cephadmlib.systemd import start_disabled_services_after_maintenance_exit
+
+        ctx = _cephadm.CephadmContext()
+        ctx.fsid = self.fsid
+        entry = self._nfs_daemon_entry(self.fsid)
+        nfs_unit = entry.identity.unit_name
+        with mock.patch('cephadmlib.systemd.daemons_matching', return_value=[entry]):
+            _call.return_value = '', '', 0
+            start_disabled_services_after_maintenance_exit(ctx)
+        _call.assert_any_call(
+            ctx, ['systemctl', 'start', nfs_unit],
+            verbosity=_cephadm.CallVerbosity.DEBUG)
+
+    @mock.patch('os.listdir', return_value=[])
+    @mock.patch('cephadmlib.systemd.daemons_matching')
+    @mock.patch('cephadmlib.systemd.call')
+    @mock.patch('cephadm.call')
+    @mock.patch('cephadm.logger')
+    @mock.patch('cephadm.systemd_target_state')
+    @mock.patch('cephadm.target_exists')
+    def test_exit_starts_disabled_services(
+            self, _target_exists, _target_state, _logger, _cephadm_call,
+            _systemd_call, _daemons_matching, _listdir):
+        entry = self._nfs_daemon_entry(TestMaintenance.fsid)
+        nfs_unit = entry.identity.unit_name
+
+        def _matching(ctx, fsid=None, daemon_type=None):
+            if daemon_type == 'nfs':
+                return [entry]
+            return []
+
+        _daemons_matching.side_effect = _matching
+        _cephadm_call.side_effect = [('', '', 0), ('', '', 0)]
+        _systemd_call.return_value = '', '', 0
+        _target_state.return_value = False
+        _target_exists.return_value = True
+        ctx: _cephadm.CephadmContext = _cephadm.cephadm_init_ctx(
+            ['host-maintenance', 'exit', '--fsid', TestMaintenance.fsid])
+        ctx.container_engine = mock_podman()
+        retval = _cephadm.change_maintenance_mode(ctx)
+        assert retval.startswith('success')
+        _systemd_call.assert_any_call(
+            ctx, ['systemctl', 'start', nfs_unit],
+            verbosity=_cephadm.CallVerbosity.DEBUG)
+
+
 class TestMonitoring(object):
     @mock.patch('cephadmlib.daemons.monitoring.call')
     def test_get_version_alertmanager(self, _call):