From 2f47f9dc3c8d8c3fc5e7f30a4a40cdd5ec3c4b0a Mon Sep 17 00:00:00 2001 From: Alexander Indenbaum Date: Sat, 23 Nov 2024 14:15:32 +0000 Subject: [PATCH] cephadm/nvmeof: support per-node gateway addresses Added gateway and discovery address maps to the service specification. These maps store per-node service addresses. The address is first searched in the map, then in the spec address configuration. If neither is defined, the host IP is used as a fallback. Signed-off-by: Alexander Indenbaum --- src/pybind/mgr/cephadm/services/nvmeof.py | 12 ++++++++++-- src/python-common/ceph/deployment/service_spec.py | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py index 0c9dc244c57..b3fd526815e 100644 --- a/src/pybind/mgr/cephadm/services/nvmeof.py +++ b/src/pybind/mgr/cephadm/services/nvmeof.py @@ -38,6 +38,8 @@ class NvmeofService(CephService): spec = cast(NvmeofServiceSpec, self.mgr.spec_store[daemon_spec.service_name].spec) nvmeof_gw_id = daemon_spec.daemon_id host_ip = self.mgr.inventory.get_addr(daemon_spec.host) + map_addr = spec.addr_map.get(daemon_spec.host) if spec.addr_map else None + map_discovery_addr = spec.discovery_addr_map.get(daemon_spec.host) if spec.discovery_addr_map else None keyring = self.get_keyring_with_caps(self.get_auth_entity(nvmeof_gw_id), ['mon', 'profile rbd', @@ -47,8 +49,14 @@ class NvmeofService(CephService): transport_tcp_options = json.dumps(spec.transport_tcp_options) if spec.transport_tcp_options else None name = '{}.{}'.format(utils.name_to_config_section('nvmeof'), nvmeof_gw_id) rados_id = name[len('client.'):] if name.startswith('client.') else name - addr = spec.addr or host_ip - discovery_addr = spec.discovery_addr or host_ip + + # The address is first searched in the per node address map, + # then in the spec address configuration. + # If neither is defined, the host IP is used as a fallback. + addr = map_addr or spec.addr or host_ip + self.mgr.log.info(f"gateway address: {addr} from {map_addr=} {spec.addr=} {host_ip=}") + discovery_addr = map_discovery_addr or spec.discovery_addr or host_ip + self.mgr.log.info(f"discovery address: {discovery_addr} from {map_discovery_addr=} {spec.discovery_addr=} {host_ip=}") context = { 'spec': spec, 'name': name, diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index e779356ae0c..fb4418595fc 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -1330,6 +1330,7 @@ class NvmeofServiceSpec(ServiceSpec): name: Optional[str] = None, group: Optional[str] = None, addr: Optional[str] = None, + addr_map: Optional[Dict[str, str]] = None, port: Optional[int] = None, pool: Optional[str] = None, enable_auth: bool = False, @@ -1380,6 +1381,7 @@ class NvmeofServiceSpec(ServiceSpec): {"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}, tgt_cmd_extra_args: Optional[str] = None, discovery_addr: Optional[str] = None, + discovery_addr_map: Optional[Dict[str, str]] = None, discovery_port: Optional[int] = None, log_level: Optional[str] = 'INFO', log_files_enabled: Optional[bool] = True, @@ -1414,6 +1416,8 @@ class NvmeofServiceSpec(ServiceSpec): self.pool = pool #: ``addr`` address of the nvmeof gateway self.addr = addr + #: ``addr_map`` per node address map of the nvmeof gateways + self.addr_map = addr_map #: ``port`` port of the nvmeof gateway self.port = port or 5500 #: ``name`` name of the nvmeof gateway @@ -1512,6 +1516,8 @@ class NvmeofServiceSpec(ServiceSpec): self.tgt_cmd_extra_args = tgt_cmd_extra_args #: ``discovery_addr`` address of the discovery service self.discovery_addr = discovery_addr + #: ``discovery_addr_map`` per node address map of the discovery service + self.discovery_addr_map = discovery_addr_map #: ``discovery_port`` port of the discovery service self.discovery_port = discovery_port or 8009 #: ``log_level`` the nvmeof gateway log level -- 2.39.5