From: Alexander Indenbaum Date: Mon, 22 Jun 2026 16:41:04 +0000 (+0300) Subject: cephadm: disable/enable NVMe-oF gateways during host maintenance X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=048d2534a6f63635aca79ad62b327cc38f0f2df4;p=ceph.git cephadm: disable/enable NVMe-oF gateways during host maintenance Automatically run nvme-gw disable before stopping daemons on maintenance enter and nvme-gw enable after daemons restart on maintenance exit so gateways fail over in a controlled way. Signed-off-by: Alexander Indenbaum --- diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 96cb2aee2d0..a9600bbac18 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2518,6 +2518,49 @@ Then run the following: self.log.info(msg) return msg + def _set_nvmeof_gateways_admin_state(self, hostname: str, enabled: bool) -> Tuple[List[str], Optional[str]]: + """Enable or disable NVMe-oF gateways on a host via nvme-gw admin state commands. + + Returns a tuple of (gateway ids updated, error message if any command failed). + """ + prefix = 'nvme-gw enable' if enabled else 'nvme-gw disable' + nvmeof_daemons = self.cache.get_daemons_by_type('nvmeof', host=hostname) + if not nvmeof_daemons: + return [], None + + updated: List[str] = [] + errors: List[str] = [] + for dd in nvmeof_daemons: + spec = cast(NvmeofServiceSpec, self.spec_store.all_specs.get(dd.service_name(), None)) + if not spec: + self.log.warning( + f'No nvmeof spec for {dd.name()}: skipping {prefix} during maintenance on {hostname}') + continue + if not spec.pool or spec.group is None: + self.log.warning( + f'nvmeof spec for {dd.service_name()} missing pool/group: ' + f'skipping {prefix} during maintenance on {hostname}') + continue + + gw_id = f'{utils.name_to_config_section("nvmeof")}.{dd.daemon_id}' + cmd = { + 'prefix': prefix, + 'id': gw_id, + 'pool': spec.pool, + 'group': spec.group, + } + self.log.info( + f'maintenance on {hostname}: {prefix} gateway {gw_id} ({spec.pool}/{spec.group})') + rc, _out, err = self.mon_command(cmd) + if rc: + errors.append(f'{gw_id}: rc={rc} {err}') + else: + updated.append(gw_id) + + if errors: + return updated, '; '.join(errors) + return updated, None + def update_maintenance_healthcheck(self) -> None: """Raise/update or clear the maintenance health check as needed""" @@ -2535,9 +2578,10 @@ Then run the following: """ Attempt to place a cluster host in maintenance Placing a host into maintenance disables the cluster's ceph target in systemd - and stops all ceph daemons. If the host is an osd host we apply the noout flag - for the host subtree in crush to prevent data movement during a host maintenance - window. + and stops all ceph daemons. NVMe-oF gateways on the host are administratively + disabled before daemons are stopped. If the host is an osd host we apply the + noout flag for the host subtree in crush to prevent data movement during a + host maintenance window. :param hostname: (str) name of the host (must match an inventory hostname) @@ -2568,6 +2612,19 @@ Then run the following: raise OrchestratorError( msg + '\nNote: Warnings can be bypassed with the --force flag', errno=rc) + if 'nvmeof' in host_daemons: + _updated, nvmeof_err = self._set_nvmeof_gateways_admin_state( + hostname, enabled=False) + if nvmeof_err: + if yes_i_really_mean_it: + self.log.warning( + f"maintenance mode request for {hostname} failed to disable " + f"some NVMe-oF gateways: {nvmeof_err}") + else: + raise OrchestratorError( + f"Unable to disable NVMe-oF gateways on {hostname}: {nvmeof_err}", + errno=errno.EIO) + # call the host-maintenance function with self.async_timeout_handler(hostname, 'cephadm host-maintenance enter'): _out, _err, _code = self.wait_async( @@ -2609,8 +2666,8 @@ Then run the following: """Exit maintenance mode and return a host to an operational state Returning from maintenance will enable the clusters systemd target and - start it, and remove any noout that has been added for the host if the - host has osd daemons + start it, re-enable any NVMe-oF gateways on the host, and remove any + noout that has been added for the host if the host has osd daemons :param hostname: (str) host name :param force: (bool) force removal of the host from maintenance mode @@ -2685,6 +2742,18 @@ Then run the following: self.log.info( f"exit maintenance request has UNSET for the noout group on host {hostname}") + if 'nvmeof' in self.cache.get_daemon_types(hostname): + _updated, nvmeof_err = self._set_nvmeof_gateways_admin_state( + hostname, enabled=True) + if nvmeof_err: + self.log.warning( + f"exit maintenance request for {hostname} failed to enable " + f"some NVMe-oF gateways: {nvmeof_err}") + if not force: + raise OrchestratorError( + f"Unable to enable NVMe-oF gateways on {hostname}: {nvmeof_err}", + errno=errno.EIO) + # update the host record status tgt_host['status'] = "" self.inventory._inventory[hostname] = tgt_host diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index 79fed21c6c4..91e320dce21 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -2298,6 +2298,80 @@ class TestCephadm(object): assert cephadm_module.inventory._inventory[hostname]['status'] == 'maintenance' + @mock.patch("cephadm.module.CephadmOrchestrator.mon_command") + @mock.patch("cephadm.serve.CephadmServe._run_cephadm") + @mock.patch("cephadm.CephadmOrchestrator._host_ok_to_stop") + @mock.patch("cephadm.module.HostCache.get_daemons_by_type") + @mock.patch("cephadm.module.HostCache.get_daemon_types") + @mock.patch("cephadm.module.HostCache.get_hosts") + def test_maintenance_enter_disables_nvmeof_gateways( + self, _hosts, _get_daemon_types, _get_daemons_by_type, _host_ok, + _run_cephadm, _mon_command, cephadm_module: CephadmOrchestrator): + hostname = 'host1' + nvmeof_daemon = DaemonDescription( + daemon_type='nvmeof', + daemon_id='pool1.grp1.host1.abcd', + hostname=hostname, + ) + _run_cephadm.side_effect = async_side_effect( + ([''], ['something\nsuccess - systemd target xxx disabled'], 0)) + _host_ok.return_value = 0, 'it is okay' + _get_daemon_types.return_value = ['nvmeof'] + _get_daemons_by_type.return_value = [nvmeof_daemon] + _hosts.return_value = [hostname, 'other_host'] + _mon_command.return_value = (0, '', '') + cephadm_module.spec_store.all_specs['nvmeof.pool1.grp1'] = ServiceSpec( + service_type='nvmeof', service_id='pool1.grp1', pool='pool1', group='grp1') + cephadm_module.inventory.add_host(HostSpec(hostname)) + + retval = cephadm_module.enter_host_maintenance(hostname) + assert retval.result_str().startswith('Daemons for Ceph cluster') + _mon_command.assert_called_once_with({ + 'prefix': 'nvme-gw disable', + 'id': 'client.nvmeof.pool1.grp1.host1.abcd', + 'pool': 'pool1', + 'group': 'grp1', + }) + _run_cephadm.assert_called_once() + + @mock.patch("cephadm.module.CephadmOrchestrator.mon_command") + @mock.patch("cephadm.serve.CephadmServe._run_cephadm") + @mock.patch("cephadm.module.HostCache.get_daemon_types") + @mock.patch("cephadm.module.HostCache.get_daemons_by_type") + @mock.patch("cephadm.module.HostCache.get_hosts") + def test_maintenance_exit_enables_nvmeof_gateways( + self, _hosts, _get_daemons_by_type, _get_daemon_types, + _run_cephadm, _mon_command, cephadm_module: CephadmOrchestrator): + hostname = 'host1' + nvmeof_daemon = DaemonDescription( + daemon_type='nvmeof', + daemon_id='pool1.grp1.host1.abcd', + hostname=hostname, + ) + _run_cephadm.side_effect = async_side_effect(([''], [ + 'something\nsuccess - systemd target xxx enabled and started'], 0)) + _get_daemon_types.return_value = ['nvmeof'] + _get_daemons_by_type.return_value = [nvmeof_daemon] + _hosts.return_value = [hostname, 'other_host'] + _mon_command.return_value = (0, '', '') + cephadm_module.spec_store.all_specs['nvmeof.pool1.grp1'] = ServiceSpec( + service_type='nvmeof', service_id='pool1.grp1', pool='pool1', group='grp1') + cephadm_module.inventory.add_host(HostSpec(hostname, status='maintenance')) + + retval = cephadm_module.exit_host_maintenance(hostname) + assert retval.result_str().startswith('Ceph cluster') + assert _run_cephadm.call_count == 2 + _run_cephadm.assert_any_call( + hostname, cephadmNoImage, 'check-host', [], error_ok=False) + _run_cephadm.assert_any_call( + hostname, cephadmNoImage, 'host-maintenance', ['exit'], error_ok=True) + _mon_command.assert_called_once_with({ + 'prefix': 'nvme-gw enable', + 'id': 'client.nvmeof.pool1.grp1.host1.abcd', + 'pool': 'pool1', + 'group': 'grp1', + }) + @mock.patch("cephadm.ssh.SSHManager._remote_connection") @mock.patch("cephadm.ssh.SSHManager._execute_command") @mock.patch("cephadm.ssh.SSHManager._check_execute_command")