self.log.info(msg)
return msg
+ def _set_nvmeof_gateways_admin_state(self, hostname: str, enabled: bool) -> Tuple[List[str], Optional[str]]:
+ """Enable or disable NVMe-oF gateways on a host via nvme-gw admin state commands.
+
+ Returns a tuple of (gateway ids updated, error message if any command failed).
+ """
+ prefix = 'nvme-gw enable' if enabled else 'nvme-gw disable'
+ nvmeof_daemons = self.cache.get_daemons_by_type('nvmeof', host=hostname)
+ if not nvmeof_daemons:
+ return [], None
+
+ updated: List[str] = []
+ errors: List[str] = []
+ for dd in nvmeof_daemons:
+ spec = cast(NvmeofServiceSpec, self.spec_store.all_specs.get(dd.service_name(), None))
+ if not spec:
+ self.log.warning(
+ f'No nvmeof spec for {dd.name()}: skipping {prefix} during maintenance on {hostname}')
+ continue
+ if not spec.pool or spec.group is None:
+ self.log.warning(
+ f'nvmeof spec for {dd.service_name()} missing pool/group: '
+ f'skipping {prefix} during maintenance on {hostname}')
+ continue
+
+ gw_id = f'{utils.name_to_config_section("nvmeof")}.{dd.daemon_id}'
+ cmd = {
+ 'prefix': prefix,
+ 'id': gw_id,
+ 'pool': spec.pool,
+ 'group': spec.group,
+ }
+ self.log.info(
+ f'maintenance on {hostname}: {prefix} gateway {gw_id} ({spec.pool}/{spec.group})')
+ rc, _out, err = self.mon_command(cmd)
+ if rc:
+ errors.append(f'{gw_id}: rc={rc} {err}')
+ else:
+ updated.append(gw_id)
+
+ if errors:
+ return updated, '; '.join(errors)
+ return updated, None
+
def update_maintenance_healthcheck(self) -> None:
"""Raise/update or clear the maintenance health check as needed"""
""" Attempt to place a cluster host in maintenance
Placing a host into maintenance disables the cluster's ceph target in systemd
- and stops all ceph daemons. If the host is an osd host we apply the noout flag
- for the host subtree in crush to prevent data movement during a host maintenance
- window.
+ and stops all ceph daemons. NVMe-oF gateways on the host are administratively
+ disabled before daemons are stopped. If the host is an osd host we apply the
+ noout flag for the host subtree in crush to prevent data movement during a
+ host maintenance window.
:param hostname: (str) name of the host (must match an inventory hostname)
raise OrchestratorError(
msg + '\nNote: Warnings can be bypassed with the --force flag', errno=rc)
+ if 'nvmeof' in host_daemons:
+ _updated, nvmeof_err = self._set_nvmeof_gateways_admin_state(
+ hostname, enabled=False)
+ if nvmeof_err:
+ if yes_i_really_mean_it:
+ self.log.warning(
+ f"maintenance mode request for {hostname} failed to disable "
+ f"some NVMe-oF gateways: {nvmeof_err}")
+ else:
+ raise OrchestratorError(
+ f"Unable to disable NVMe-oF gateways on {hostname}: {nvmeof_err}",
+ errno=errno.EIO)
+
# call the host-maintenance function
with self.async_timeout_handler(hostname, 'cephadm host-maintenance enter'):
_out, _err, _code = self.wait_async(
"""Exit maintenance mode and return a host to an operational state
Returning from maintenance will enable the clusters systemd target and
- start it, and remove any noout that has been added for the host if the
- host has osd daemons
+ start it, re-enable any NVMe-oF gateways on the host, and remove any
+ noout that has been added for the host if the host has osd daemons
:param hostname: (str) host name
:param force: (bool) force removal of the host from maintenance mode
self.log.info(
f"exit maintenance request has UNSET for the noout group on host {hostname}")
+ if 'nvmeof' in self.cache.get_daemon_types(hostname):
+ _updated, nvmeof_err = self._set_nvmeof_gateways_admin_state(
+ hostname, enabled=True)
+ if nvmeof_err:
+ self.log.warning(
+ f"exit maintenance request for {hostname} failed to enable "
+ f"some NVMe-oF gateways: {nvmeof_err}")
+ if not force:
+ raise OrchestratorError(
+ f"Unable to enable NVMe-oF gateways on {hostname}: {nvmeof_err}",
+ errno=errno.EIO)
+
# update the host record status
tgt_host['status'] = ""
self.inventory._inventory[hostname] = tgt_host
assert cephadm_module.inventory._inventory[hostname]['status'] == 'maintenance'
+ @mock.patch("cephadm.module.CephadmOrchestrator.mon_command")
+ @mock.patch("cephadm.serve.CephadmServe._run_cephadm")
+ @mock.patch("cephadm.CephadmOrchestrator._host_ok_to_stop")
+ @mock.patch("cephadm.module.HostCache.get_daemons_by_type")
+ @mock.patch("cephadm.module.HostCache.get_daemon_types")
+ @mock.patch("cephadm.module.HostCache.get_hosts")
+ def test_maintenance_enter_disables_nvmeof_gateways(
+ self, _hosts, _get_daemon_types, _get_daemons_by_type, _host_ok,
+ _run_cephadm, _mon_command, cephadm_module: CephadmOrchestrator):
+ hostname = 'host1'
+ nvmeof_daemon = DaemonDescription(
+ daemon_type='nvmeof',
+ daemon_id='pool1.grp1.host1.abcd',
+ hostname=hostname,
+ )
+ _run_cephadm.side_effect = async_side_effect(
+ ([''], ['something\nsuccess - systemd target xxx disabled'], 0))
+ _host_ok.return_value = 0, 'it is okay'
+ _get_daemon_types.return_value = ['nvmeof']
+ _get_daemons_by_type.return_value = [nvmeof_daemon]
+ _hosts.return_value = [hostname, 'other_host']
+ _mon_command.return_value = (0, '', '')
+ cephadm_module.spec_store.all_specs['nvmeof.pool1.grp1'] = ServiceSpec(
+ service_type='nvmeof', service_id='pool1.grp1', pool='pool1', group='grp1')
+ cephadm_module.inventory.add_host(HostSpec(hostname))
+
+ retval = cephadm_module.enter_host_maintenance(hostname)
+ assert retval.result_str().startswith('Daemons for Ceph cluster')
+ _mon_command.assert_called_once_with({
+ 'prefix': 'nvme-gw disable',
+ 'id': 'client.nvmeof.pool1.grp1.host1.abcd',
+ 'pool': 'pool1',
+ 'group': 'grp1',
+ })
+ _run_cephadm.assert_called_once()
+
+ @mock.patch("cephadm.module.CephadmOrchestrator.mon_command")
+ @mock.patch("cephadm.serve.CephadmServe._run_cephadm")
+ @mock.patch("cephadm.module.HostCache.get_daemon_types")
+ @mock.patch("cephadm.module.HostCache.get_daemons_by_type")
+ @mock.patch("cephadm.module.HostCache.get_hosts")
+ def test_maintenance_exit_enables_nvmeof_gateways(
+ self, _hosts, _get_daemons_by_type, _get_daemon_types,
+ _run_cephadm, _mon_command, cephadm_module: CephadmOrchestrator):
+ hostname = 'host1'
+ nvmeof_daemon = DaemonDescription(
+ daemon_type='nvmeof',
+ daemon_id='pool1.grp1.host1.abcd',
+ hostname=hostname,
+ )
+ _run_cephadm.side_effect = async_side_effect(([''], [
+ 'something\nsuccess - systemd target xxx enabled and started'], 0))
+ _get_daemon_types.return_value = ['nvmeof']
+ _get_daemons_by_type.return_value = [nvmeof_daemon]
+ _hosts.return_value = [hostname, 'other_host']
+ _mon_command.return_value = (0, '', '')
+ cephadm_module.spec_store.all_specs['nvmeof.pool1.grp1'] = ServiceSpec(
+ service_type='nvmeof', service_id='pool1.grp1', pool='pool1', group='grp1')
+ cephadm_module.inventory.add_host(HostSpec(hostname, status='maintenance'))
+
+ retval = cephadm_module.exit_host_maintenance(hostname)
+ assert retval.result_str().startswith('Ceph cluster')
+ assert _run_cephadm.call_count == 2
+ _run_cephadm.assert_any_call(
+ hostname, cephadmNoImage, 'check-host', [], error_ok=False)
+ _run_cephadm.assert_any_call(
+ hostname, cephadmNoImage, 'host-maintenance', ['exit'], error_ok=True)
+ _mon_command.assert_called_once_with({
+ 'prefix': 'nvme-gw enable',
+ 'id': 'client.nvmeof.pool1.grp1.host1.abcd',
+ 'pool': 'pool1',
+ 'group': 'grp1',
+ })
+
@mock.patch("cephadm.ssh.SSHManager._remote_connection")
@mock.patch("cephadm.ssh.SSHManager._execute_command")
@mock.patch("cephadm.ssh.SSHManager._check_execute_command")
projects / ceph.git / commitdiff