From: Redouane Kachach Date: Wed, 3 Dec 2025 09:36:25 +0000 (+0100) Subject: mgr/cephadm: fix nvmeof TLS handling and add coverage for ssl/mTLS X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8072940c32e54900fe852a37fc455c6c6d7009f4;p=ceph.git mgr/cephadm: fix nvmeof TLS handling and add coverage for ssl/mTLS This PR fixes the value of `ssl` field on `NvmeofServiceSpec` (was always set to enable_auth) and add some UT to make sure both specs with ssl only and with mTLS enabled (enable_auth) generate the expected daemon configuration. Fixes: https://tracker.ceph.com/issues/74073 Signed-off-by: Redouane Kachach --- diff --git a/src/pybind/mgr/cephadm/tests/services/test_nvmeof.py b/src/pybind/mgr/cephadm/tests/services/test_nvmeof.py index 1b45390a9075..b255f92531fd 100644 --- a/src/pybind/mgr/cephadm/tests/services/test_nvmeof.py +++ b/src/pybind/mgr/cephadm/tests/services/test_nvmeof.py @@ -8,6 +8,7 @@ from cephadm.module import CephadmOrchestrator from ceph.deployment.service_spec import NvmeofServiceSpec from cephadm.tests.fixtures import with_host, with_service, _run_cephadm, async_side_effect from orchestrator import OrchestratorError +from cephadm.tlsobject_types import TLSCredentials class FakeInventory: @@ -301,3 +302,386 @@ timeout = 1.0\n""" spec = NvmeofServiceSpec(service_id='pool4.bla.group', group='group') assert spec.pool == ".nvmeof" + + @patch("cephadm.inventory.Inventory.get_addr", lambda _, __: '192.168.100.100') + @patch("cephadm.serve.CephadmServe._run_cephadm") + @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", + lambda instance, dspec, ips=None, fqdns=None: TLSCredentials('mycert', 'mykey'), + ) + @patch("cephadm.services.nvmeof.NvmeofService.get_self_signed_certificates_with_label", + lambda instance, spec, daemon_spec, label: TLSCredentials('client_cert', 'client_key', 'nvmeof_root_ca'), + ) + @patch("cephadm.module.CephadmOrchestrator.get_unique_name") + def test_nvmeof_config_when_ssl_and_auth_enabled( + self, + _get_name, + _run_cephadm, + cephadm_module: CephadmOrchestrator, + ): + """ + When ssl=True and enable_auth=True are set on NvmeofServiceSpec, the generated + ceph-nvmeof.conf must have enable_auth = True, and config_blobs.files must + include server_cert/server_key as well as client_cert/client_key/root_ca_cert. + + This ensures mTLS wiring (configure_tls) is correct. + """ + + pool = 'testpool' + group = 'mygroup' + nvmeof_daemon_id = f'{pool}.{group}.test.qwert' + tgt_cmd_extra_args = '--cpumask=0xFF --msg-mempool-size=524288' + default_port = 5500 + + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + _get_name.return_value = nvmeof_daemon_id + + # Expected config when ssl=True and enable_auth=True. + nvmeof_gateway_conf_mtls = f"""# This file is generated by cephadm. +[gateway] +name = client.nvmeof.{nvmeof_daemon_id} +group = {group} +addr = 192.168.100.100 +port = {default_port} +enable_auth = True +state_update_notify = True +state_update_interval_sec = 5 +break_update_interval_sec = 25 +enable_spdk_discovery_controller = False +encryption_key = /encryption.key +rebalance_period_sec = 7 +max_gws_in_grp = 16 +max_ns_to_change_lb_grp = 8 +enable_prometheus_exporter = True +prometheus_exporter_ssl = False +prometheus_port = 10008 +prometheus_stats_interval = 10 +prometheus_frequency_slow_down_factor = 3.0 +prometheus_cycles_to_adjust_speed = 3 +prometheus_startup_delay = 240 +prometheus_connection_list_cache_expiration = 60 +verify_nqns = True +verify_keys = True +verify_listener_ip = True +# This is a development flag, do not change it +abort_on_errors = True +# This is a development flag, do not change it +abort_on_update_error = True +# This is a development flag, do not change it +omap_file_ignore_unlock_errors = False +# This is a development flag, do not change it +omap_file_lock_on_read = True +omap_file_lock_duration = 40 +omap_file_lock_retries = 30 +omap_file_lock_retry_sleep_interval = 1.0 +omap_file_update_reloads = 10 +omap_file_update_attempts = 500 +allowed_consecutive_spdk_ping_failures = 1 +spdk_ping_interval_in_seconds = 2.0 +ping_spdk_under_lock = False +enable_monitor_client = True +max_hosts_per_namespace = 16 +max_namespaces_with_netmask = 1000 +max_subsystems = 128 +max_hosts = 2048 +max_namespaces = 4096 +max_namespaces_per_subsystem = 512 +max_hosts_per_subsystem = 128 +subsystem_cache_expiration = 30 +force_tls = False +# This is a development flag, do not change it +max_message_length_in_mb = 4 +io_stats_enabled = True + +[gateway-logs] +log_level = INFO +log_files_enabled = True +log_files_rotation_enabled = True +verbose_log_messages = True +max_log_file_size_in_mb = 10 +max_log_files_count = 20 +max_log_directory_backups = 10 +log_directory = /var/log/ceph/ + +[discovery] +addr = 192.168.100.100 +port = 8009 +# This is a development flag, do not change it +abort_on_errors = True +bind_retries_limit = 10 +bind_sleep_interval = 0.5 + +[ceph] +pool = {pool} +config_file = /etc/ceph/ceph.conf +id = nvmeof.{nvmeof_daemon_id} + +[mtls] +server_key = /server.key +client_key = /client.key +server_cert = /server.cert +client_cert = /client.cert +root_ca_cert = /root.ca.cert + +[kmip] +cert_dir = ./certs/kmip/{{server_name}} + +[spdk] +tgt_path = /usr/local/bin/nvmf_tgt +rpc_socket_dir = /var/tmp/ +rpc_socket_name = spdk.sock +timeout = 60.0 +cluster_connections = 32 +protocol_log_level = WARNING +conn_retries = 10 +transports = tcp +transport_tcp_options = {{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}} +enable_dsa_acceleration = False +rbd_with_crc32c = True +tgt_cmd_extra_args = {tgt_cmd_extra_args} +qos_timeslice_in_usecs = 0 +notifications_interval = 60 + +[monitor] +timeout = 1.0 +""" + + with with_host(cephadm_module, 'test'): + with with_service( + cephadm_module, + NvmeofServiceSpec( + service_id=f'{pool}.{group}', + tgt_cmd_extra_args=tgt_cmd_extra_args, + group=group, + pool=pool, + ssl=True, + enable_auth=True + ), + ): + _run_cephadm.assert_called_with( + 'test', + f'nvmeof.{nvmeof_daemon_id}', + ['_orch', 'deploy'], + [], + stdin=json.dumps({ + "fsid": "fsid", + "name": f"nvmeof.{nvmeof_daemon_id}", + "image": "", + "deploy_arguments": [], + "params": { + "tcp_ports": [5500, 4420, 8009, 10008], + }, + "meta": { + "service_name": "nvmeof.testpool.mygroup", + "ports": [5500, 4420, 8009, 10008], + "ip": None, + "deployed_by": [], + "rank": None, + "rank_generation": None, + "extra_container_args": None, + "extra_entrypoint_args": None, + }, + "config_blobs": { + "config": "", + "keyring": "[client.nvmeof.testpool.mygroup.test.qwert]\nkey = None\n", + "files": { + # server-side TLS + "server_cert": "mycert", + "server_key": "mykey", + # mTLS client-side pieces from get_self_signed_certificates_with_label + "client_cert": "client_cert", + "client_key": "client_key", + "root_ca_cert": "nvmeof_root_ca", + "ceph-nvmeof.conf": nvmeof_gateway_conf_mtls, + }, + }, + }), + error_ok=True, + use_current_daemon_image=False, + ) + + @patch("cephadm.inventory.Inventory.get_addr", lambda _, __: '192.168.100.100') + @patch("cephadm.serve.CephadmServe._run_cephadm") + @patch("cephadm.cert_mgr.CertMgr.get_root_ca", lambda instance: 'nvmeof_root_cert') + @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", + lambda instance, dspec, ips=None, fqdns=None: TLSCredentials('mycert', 'mykey'), + ) + @patch("cephadm.module.CephadmOrchestrator.get_unique_name") + def test_nvmeof_config_when_ssl_enabled( + self, + _get_name, + _run_cephadm, + cephadm_module: CephadmOrchestrator, + ): + """ + When ssl=True is set on NvmeofServiceSpec, the generated ceph-nvmeof.conf + must enable TLS, and the generated certificate files must be passed in config_blobs. + """ + + pool = 'testpool' + group = 'mygroup' + nvmeof_daemon_id = f'{pool}.{group}.test.qwert' + tgt_cmd_extra_args = '--cpumask=0xFF --msg-mempool-size=524288' + default_port = 5500 + + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + _get_name.return_value = nvmeof_daemon_id + + # Expected config when ssl=True: + # note: enable_auth stays False on purpose in this case + nvmeof_gateway_conf_ssl = f"""# This file is generated by cephadm. +[gateway] +name = client.nvmeof.{nvmeof_daemon_id} +group = {group} +addr = 192.168.100.100 +port = {default_port} +enable_auth = False +state_update_notify = True +state_update_interval_sec = 5 +break_update_interval_sec = 25 +enable_spdk_discovery_controller = False +encryption_key = /encryption.key +rebalance_period_sec = 7 +max_gws_in_grp = 16 +max_ns_to_change_lb_grp = 8 +enable_prometheus_exporter = True +prometheus_exporter_ssl = False +prometheus_port = 10008 +prometheus_stats_interval = 10 +prometheus_frequency_slow_down_factor = 3.0 +prometheus_cycles_to_adjust_speed = 3 +prometheus_startup_delay = 240 +prometheus_connection_list_cache_expiration = 60 +verify_nqns = True +verify_keys = True +verify_listener_ip = True +# This is a development flag, do not change it +abort_on_errors = True +# This is a development flag, do not change it +abort_on_update_error = True +# This is a development flag, do not change it +omap_file_ignore_unlock_errors = False +# This is a development flag, do not change it +omap_file_lock_on_read = True +omap_file_lock_duration = 40 +omap_file_lock_retries = 30 +omap_file_lock_retry_sleep_interval = 1.0 +omap_file_update_reloads = 10 +omap_file_update_attempts = 500 +allowed_consecutive_spdk_ping_failures = 1 +spdk_ping_interval_in_seconds = 2.0 +ping_spdk_under_lock = False +enable_monitor_client = True +max_hosts_per_namespace = 16 +max_namespaces_with_netmask = 1000 +max_subsystems = 128 +max_hosts = 2048 +max_namespaces = 4096 +max_namespaces_per_subsystem = 512 +max_hosts_per_subsystem = 128 +subsystem_cache_expiration = 30 +force_tls = False +# This is a development flag, do not change it +max_message_length_in_mb = 4 +io_stats_enabled = True + +[gateway-logs] +log_level = INFO +log_files_enabled = True +log_files_rotation_enabled = True +verbose_log_messages = True +max_log_file_size_in_mb = 10 +max_log_files_count = 20 +max_log_directory_backups = 10 +log_directory = /var/log/ceph/ + +[discovery] +addr = 192.168.100.100 +port = 8009 +# This is a development flag, do not change it +abort_on_errors = True +bind_retries_limit = 10 +bind_sleep_interval = 0.5 + +[ceph] +pool = {pool} +config_file = /etc/ceph/ceph.conf +id = nvmeof.{nvmeof_daemon_id} + +[mtls] +server_key = /server.key +client_key = /client.key +server_cert = /server.cert +client_cert = /client.cert +root_ca_cert = /root.ca.cert + +[kmip] +cert_dir = ./certs/kmip/{{server_name}} + +[spdk] +tgt_path = /usr/local/bin/nvmf_tgt +rpc_socket_dir = /var/tmp/ +rpc_socket_name = spdk.sock +timeout = 60.0 +cluster_connections = 32 +protocol_log_level = WARNING +conn_retries = 10 +transports = tcp +transport_tcp_options = {{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}} +enable_dsa_acceleration = False +rbd_with_crc32c = True +tgt_cmd_extra_args = {tgt_cmd_extra_args} +qos_timeslice_in_usecs = 0 +notifications_interval = 60 + +[monitor] +timeout = 1.0 +""" + + with with_host(cephadm_module, 'test'): + with with_service( + cephadm_module, + NvmeofServiceSpec( + service_id=f'{pool}.{group}', + tgt_cmd_extra_args=tgt_cmd_extra_args, + group=group, + pool=pool, + ssl=True, + ), + ): + _run_cephadm.assert_called_with( + 'test', + f'nvmeof.{nvmeof_daemon_id}', + ['_orch', 'deploy'], + [], + stdin=json.dumps({ + "fsid": "fsid", + "name": f"nvmeof.{nvmeof_daemon_id}", + "image": "", + "deploy_arguments": [], + "params": { + "tcp_ports": [5500, 4420, 8009, 10008], + }, + "meta": { + "service_name": "nvmeof.testpool.mygroup", + "ports": [5500, 4420, 8009, 10008], + "ip": None, + "deployed_by": [], + "rank": None, + "rank_generation": None, + "extra_container_args": None, + "extra_entrypoint_args": None, + }, + "config_blobs": { + "config": "", + "keyring": "[client.nvmeof.testpool.mygroup.test.qwert]\nkey = None\n", + "files": { + # certs come from the mocked get_certificates() + "server_cert": "mycert", + "server_key": "mykey", + "ceph-nvmeof.conf": nvmeof_gateway_conf_ssl, + }, + }, + }), + error_ok=True, + use_current_daemon_image=False, + ) diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 3e7aada7319f..5029573252ad 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -1787,7 +1787,7 @@ class NvmeofServiceSpec(ServiceSpec): self.group = group or '' #: ``enable_auth`` enables user authentication on nvmeof gateway self.enable_auth = enable_auth - self.ssl = enable_auth # to force enabling ssl field when auth is enabled + self.ssl = ssl or enable_auth # to force enabling ssl field when auth is enabled #: ``state_update_notify`` enables automatic update from OMAP in nvmeof gateway self.state_update_notify = state_update_notify #: ``state_update_interval_sec`` number of seconds to check for updates in OMAP