From: Shubha Jain Date: Wed, 14 Jan 2026 17:15:59 +0000 (+0530) Subject: mgr/cephadm,nfs: support monitoring_addr and bind_addr for NFS X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f70b3a29344ba67e79dc8f24eae5767456da6e26;p=ceph.git mgr/cephadm,nfs: support monitoring_addr and bind_addr for NFS Add support for configuring monitoring_addr and bind_addr for NFS daemons managed by cephadm. Also restores required RADOS configuration blocks and updates tests accordingly. Fixes: https://tracker.ceph.com/issues/74403 Signed-off-by: Shubha Jain --- diff --git a/src/pybind/mgr/cephadm/services/nfs.py b/src/pybind/mgr/cephadm/services/nfs.py index 8227e030965..27641ab588d 100644 --- a/src/pybind/mgr/cephadm/services/nfs.py +++ b/src/pybind/mgr/cephadm/services/nfs.py @@ -173,13 +173,16 @@ class NFSService(CephService): rgw_user = f'{rados_user}-rgw' rgw_keyring = self.create_rgw_keyring(daemon_spec) bind_addr = '' + if spec.virtual_ip and not spec.enable_haproxy_protocol: + # keepalive_only mode: prioritize virtual_ip bind_addr = spec.virtual_ip daemon_spec.port_ips = {str(port): spec.virtual_ip} # update daemon spec ip for prometheus, as monitoring will happen on this # ip, if no monitor ip specified daemon_spec.ip = bind_addr elif daemon_spec.ip: + # daemon_spec.ip is already set by scheduler from ip_addrs if specified bind_addr = daemon_spec.ip daemon_spec.port_ips = {str(port): daemon_spec.ip} if not bind_addr: @@ -563,9 +566,26 @@ class NFSService(CephService): # check if monitor needs to be bind on specific ip monitoring_addr = spec.monitoring_ip_addrs.get(host) if spec.monitoring_ip_addrs else None - if monitoring_addr and monitoring_addr not in self.mgr.cache.get_host_network_ips(host): - logger.debug(f"Monitoring IP {monitoring_addr} is not configured on host {host}.") - monitoring_addr = None + + if monitoring_addr: + try: + ip = ipaddress.ip_address(monitoring_addr) + + # Fetch host IPs once and normalize to strings + host_ips = set(self.mgr.cache.get_host_network_ips(host)) + + # Allow loopback addresses without requiring them on an interface + if not ip.is_loopback and str(ip) not in host_ips: + logger.debug( + f"Monitoring IP {monitoring_addr} is not configured on host {host}." + ) + monitoring_addr = None + + except ValueError: + logger.warning( + f"Invalid monitoring IP address {monitoring_addr} for host {host}." + ) + monitoring_addr = None if not monitoring_addr and spec.monitoring_networks: monitoring_addr = self.mgr.get_first_matching_network_ip(host, spec, spec.monitoring_networks) if not monitoring_addr: diff --git a/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 index 2e2461c0841..768e64302f4 100644 --- a/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 +++ b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 @@ -11,16 +11,18 @@ NFS_CORE_PARAM { Enable_UDP = false; NFS_Port = {{ port }}; allow_set_io_flusher_fail = true; +{% if monitoring_addr %} + Monitoring_Addr = {{ monitoring_addr }}; +{% endif %} +{% if monitoring_port %} + Monitoring_Port = {{ monitoring_port }}; +{% endif %} {% if bind_addr %} Bind_addr = {{ bind_addr }}; {% endif %} {% if haproxy_hosts %} HAProxy_Hosts = {{ haproxy_hosts|join(", ") }}; {% endif %} -{% if monitoring_addr %} - Monitoring_Addr = {{ monitoring_addr }}; -{% endif %} - Monitoring_Port = {{ monitoring_port }}; {% if enable_rdma and rdma_port %} NFS_RDMA_Port = {{ rdma_port }}; {% endif %} @@ -78,19 +80,19 @@ TLS_CONFIG{ TLS_Cert_File = /etc/ganesha/tls/tls_cert.pem; TLS_Key_File = /etc/ganesha/tls/tls_key.pem; TLS_CA_File = /etc/ganesha/tls/tls_ca_cert.pem; - {% if tls_ciphers %} +{% if tls_ciphers %} TLS_Ciphers = "{{ tls_ciphers }}"; - {% endif %} - {% if tls_min_version %} +{% endif %} +{% if tls_min_version %} TLS_Min_Version = "{{ tls_min_version }}"; - {% endif %} - {% if tls_ktls %} +{% endif %} +{% if tls_ktls %} Enable_KTLS = {{ tls_ktls }}; - {% endif %} - {% if tls_debug %} +{% endif %} +{% if tls_debug %} Enable_debug = {{ tls_debug }}; - {% endif %} +{% endif %} } - {% endif %} + %url {{ url }} diff --git a/src/pybind/mgr/nfs/cluster.py b/src/pybind/mgr/nfs/cluster.py index d67ca4cb1e3..acbedf104af 100644 --- a/src/pybind/mgr/nfs/cluster.py +++ b/src/pybind/mgr/nfs/cluster.py @@ -170,6 +170,9 @@ class NFSCluster: tls_debug: bool = False, tls_min_version: Optional[str] = None, tls_ciphers: Optional[str] = None, + ip_addrs: Optional[Dict[str, str]] = None, + monitoring_ip_addrs: Optional[Dict[str, str]] = None, + monitoring_port: Optional[int] = None, enable_rdma: bool = False, rdma_port: Optional[int] = None, ) -> None: @@ -216,6 +219,9 @@ class NFSCluster: tls_debug=tls_debug, tls_min_version=tls_min_version, tls_ciphers=tls_ciphers, + ip_addrs=ip_addrs, + monitoring_ip_addrs=monitoring_ip_addrs, + monitoring_port=monitoring_port, enable_rdma=enable_rdma, rdma_port=rdma_port) completion = self.mgr.apply_nfs(spec) @@ -246,6 +252,9 @@ class NFSCluster: tls_debug=tls_debug, tls_min_version=tls_min_version, tls_ciphers=tls_ciphers, + ip_addrs=ip_addrs, + monitoring_ip_addrs=monitoring_ip_addrs, + monitoring_port=monitoring_port, enable_rdma=enable_rdma, rdma_port=rdma_port) completion = self.mgr.apply_nfs(spec) @@ -281,6 +290,9 @@ class NFSCluster: tls_debug: bool = False, tls_min_version: Optional[str] = None, tls_ciphers: Optional[str] = None, + ip_addrs: Optional[Dict[str, str]] = None, + monitoring_ip_addrs: Optional[Dict[str, str]] = None, + monitoring_port: Optional[int] = None, enable_rdma: bool = False, rdma_port: Optional[int] = None, ) -> None: @@ -322,6 +334,9 @@ class NFSCluster: tls_debug=tls_debug, tls_min_version=tls_min_version, tls_ciphers=tls_ciphers, + ip_addrs=ip_addrs, + monitoring_ip_addrs=monitoring_ip_addrs, + monitoring_port=monitoring_port, enable_rdma=enable_rdma, rdma_port=rdma_port ) @@ -402,7 +417,6 @@ class NFSCluster: if len(svc.ports) > 1: monitor_port = svc.ports[1] except orchestrator.OrchestratorError: - # No ingress service found for this cluster log.debug(f"No ingress service found for NFS cluster {cluster_id}") # Build backend list with daemon information @@ -412,17 +426,13 @@ class NFSCluster: continue try: - # Resolve daemon IP if daemon.ip: ip = daemon.ip elif daemon.hostname in hosts_map: - # Use cached host data ip = resolve_ip(hosts_map[daemon.hostname].addr) else: - # Fallback to hostname resolution ip = resolve_ip(daemon.hostname) - # Get daemon status status = orchestrator.DaemonDescriptionStatus.to_str(daemon.status) backends.append({ @@ -437,14 +447,11 @@ class NFSCluster: f" on {daemon.hostname} in cluster {cluster_id}") continue - # Sort backends by hostname for consistent output backends.sort(key=lambda x: x["hostname"]) - # Determine deployment type based on ingress configuration and actual daemon count deployment_type = "standalone" placement = None - # Get NFS service spec for placement information first nfs_sc = self.mgr.describe_service( service_type='nfs', service_name=f'nfs.{cluster_id}' @@ -456,17 +463,13 @@ class NFSCluster: break if ingress_mode: - # Determine deployment type from placement spec (source of truth) - # Note: Using placement.count instead of len(backends) to avoid race conditions if placement and placement.count and placement.count > 1: deployment_type = "active-active" elif len(backends) > 1: - # Fallback to actual daemon count if placement.count not set deployment_type = "active-active" else: deployment_type = "active-passive" - # Build result dictionary r: Dict[str, Any] = { 'deployment_type': deployment_type, 'virtual_ip': virtual_ip, @@ -474,7 +477,6 @@ class NFSCluster: 'placement': placement.to_json() if placement else {}, } - # Add ingress configuration to result if ingress_mode: r['ingress_mode'] = ingress_mode.value if ingress_port is not None: diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 0f4ed118d94..80d5fb46855 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -162,6 +162,9 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): virtual_ip: Optional[str] = None, ingress_mode: Optional[IngressType] = None, port: Optional[int] = None, + bind_addrs: Optional[str] = None, + monitoring_addrs: Optional[str] = None, + monitoring_port: Optional[int] = None, enable_rdma: bool = False, rdma_port: Optional[int] = None, enable_nfsv3: bool = False, @@ -182,11 +185,31 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): tls_debug = config.get('tls_debug') tls_ciphers = config.get('tls_ciphers') + # Parse bind_addrs and monitoring_addrs from CLI format + ip_addrs = None + if bind_addrs: + ip_addrs = {} + for pair in bind_addrs.split(','): + if ':' in pair: + host, ip = pair.split(':', 1) + ip_addrs[host.strip()] = ip.strip() + + monitoring_ip_addrs = None + if monitoring_addrs: + monitoring_ip_addrs = {} + for pair in monitoring_addrs.split(','): + if ':' in pair: + host, ip = pair.split(':', 1) + monitoring_ip_addrs[host.strip()] = ip.strip() + return self.nfs.create_nfs_cluster(cluster_id=cluster_id, placement=placement, virtual_ip=virtual_ip, ingress=ingress, ingress_mode=ingress_mode, port=port, cluster_qos_config=cluster_qos_config, enable_nfsv3=enable_nfsv3, + ip_addrs=ip_addrs, + monitoring_ip_addrs=monitoring_ip_addrs, + monitoring_port=monitoring_port, ssl=ssl, ssl_cert=ssl_cert, ssl_key=ssl_key,