From: Shweta Bhosale Date: Fri, 30 May 2025 16:26:31 +0000 (+0530) Subject: mgr/cephadm: During upgrade if NFS cluster is using old nodeids, keep using it X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=12e2928f81b0cf61b4d6d4e59653b8e7e3dddfd5;p=ceph.git mgr/cephadm: During upgrade if NFS cluster is using old nodeids, keep using it Fixes: https://tracker.ceph.com/issues/71509 Signed-off-by: Shweta Bhosale --- diff --git a/src/pybind/mgr/cephadm/migrations.py b/src/pybind/mgr/cephadm/migrations.py index d2831ec46da..315c1105d3c 100644 --- a/src/pybind/mgr/cephadm/migrations.py +++ b/src/pybind/mgr/cephadm/migrations.py @@ -1,11 +1,13 @@ import json import re import logging -from typing import TYPE_CHECKING, Iterator, Optional, Dict, Any, List +from typing import TYPE_CHECKING, Iterator, Optional, Dict, Any, List, cast -from ceph.deployment.service_spec import PlacementSpec, ServiceSpec, HostPlacementSpec, RGWSpec, CertificateSource +from ceph.deployment.service_spec import PlacementSpec, ServiceSpec, HostPlacementSpec, RGWSpec, CertificateSource, NFSServiceSpec from cephadm.schedule import HostAssignment from cephadm.utils import SpecialHostLabels +from cephadm.services.nfs import NFSService +from cephadm.services.service_registry import service_registry import rados from mgr_util import get_cert_issuer_info @@ -15,7 +17,7 @@ from orchestrator import OrchestratorError, DaemonDescription if TYPE_CHECKING: from .module import CephadmOrchestrator -LAST_MIGRATION = 8 +LAST_MIGRATION = 9 logger = logging.getLogger(__name__) @@ -122,6 +124,11 @@ class Migrations: if self.migrate_7_8(): self.set(8) + if self.mgr.migration_current == 8 and not startup: + logger.info('Running migration 8 -> 9') + if self.migrate_8_9(): + self.set(9) + def migrate_0_1(self) -> bool: """ Migration 0 -> 1 @@ -530,6 +537,30 @@ class Migrations: logger.error(f"Promtail -> Alloy migration failed: {e}") return False + def migrate_8_9(self) -> bool: + # For NFS, check if nfs cluster is using old types of node ids (service_name.0) + # if yes, then store those daemons in mon store, to continue using old node ids for those daemons + nfs_services = [] + service_specs = self.mgr.spec_store.get_specs_by_type('nfs') + nfs_service = cast(NFSService, service_registry.get_service('nfs')) + try: + for service_name, spec in service_specs.items(): + # get grace tool dump + out = nfs_service.run_grace_tool(cast(NFSServiceSpec, spec), 'dump') + if service_name in out: + nfs_services.append(service_name) + logger.info(f'NFS service {nfs_service} needs to maintain old node ids after upgrade') + except Exception as e: + logger.exception(f'Got error while executing grace tool: {e}') + self.mgr.set_health_warning('CEPHADM_MIGRATION_FAILURE', + f'Cephadm migration failed: {e}', + 1, [str(e)]) + return False + if nfs_services: + self.mgr.set_store('nfs_services_with_old_nodeid', ','.join(nfs_services)) + self.mgr.remove_health_warning('CEPHADM_MIGRATION_FAILURE') + return True + def queue_migrate_rgw_spec(mgr: "CephadmOrchestrator", spec_dict: Dict[Any, Any]) -> None: """ diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 1fb853712ad..f8e81985e67 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -3265,9 +3265,16 @@ Then run the following: for h, ls in osds_msg.items(): msg += f'\thost {h}: {" ".join([f"osd.{id}" for id in ls])}' raise OrchestratorError( - f'If {service_name} is removed then the following OSDs ' - f'will remain, --force to proceed anyway\n{msg}' - ) + f'If {service_name} is removed then the following OSDs will remain, --force to proceed anyway\n{msg}') + elif service_name.startswith('nfs.'): + # check if its using old node id style and remove from mon store + nfs_services = self.get_store('nfs_services_with_old_nodeid') + if nfs_services: + nfs_services = nfs_services.split(',') + if service_name in nfs_services: + nfs_services.remove(service_name) + val = ','.join(nfs_services) if nfs_services else None + self.set_store('nfs_services_with_old_nodeid', val) spec = self.spec_store[service_name].spec CephadmServe(self)._remove_service_config(spec) diff --git a/src/pybind/mgr/cephadm/services/nfs.py b/src/pybind/mgr/cephadm/services/nfs.py index 27641ab588d..7cdb3ce1b56 100644 --- a/src/pybind/mgr/cephadm/services/nfs.py +++ b/src/pybind/mgr/cephadm/services/nfs.py @@ -92,7 +92,7 @@ class NFSService(CephService): for daemon_id in m.values(): if daemon_id is not None: self.fence(daemon_id) - nodeid = f'{rank}' + nodeid = self.get_daemon_nodeid(spec.service_name(), rank) self.mgr.log.info( "Removing %s from the ganesha grace table for service %s", nodeid, service_name ) @@ -144,6 +144,12 @@ class NFSService(CephService): daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec) return daemon_spec + def get_daemon_nodeid(self, service_name: str, rank: Optional[int]) -> str: + out = self.mgr.get_store('nfs_services_with_old_nodeid') + if out and service_name in out.split(','): + return f'{service_name}.{rank}' + return str(rank) + def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]: assert self.TYPE == daemon_spec.daemon_type super().prepare_certificates(daemon_spec) @@ -152,7 +158,7 @@ class NFSService(CephService): host = daemon_spec.host spec = cast(NFSServiceSpec, self.mgr.spec_store[daemon_spec.service_name].spec) - nodeid = f'{daemon_spec.rank}' + nodeid = self.get_daemon_nodeid(spec.service_name(), daemon_spec.rank) nfs_idmap_conf = '/etc/ganesha/idmap.conf' @@ -266,7 +272,8 @@ class NFSService(CephService): "tls_ktls": spec.tls_ktls, "tls_debug": spec.tls_debug, "ceph_nodes": ceph_nodes, - "protocols": "3, 4" if spec.enable_nfsv3 else "4" + "protocols": "3, 4" if spec.enable_nfsv3 else "4", + "use_old_nodeid": False if nodeid.isdigit() else True } if spec.enable_haproxy_protocol: context["haproxy_hosts"] = self._haproxy_hosts() @@ -403,7 +410,7 @@ class NFSService(CephService): def run_grace_tool(self, spec: NFSServiceSpec, action: str, - nodeid: str) -> None: + nodeid: str = '') -> str: # write a temp keyring and referencing config file. this is a kludge # because the ganesha-grace-tool can only authenticate as a client (and # not a mgr). Also, it doesn't allow you to pass a keyring location via @@ -445,9 +452,10 @@ class NFSService(CephService): 'Ignore ganesha-rados-grace tool remove failure as %s does not exists for %s service', nodeid, spec.service_name() ) - return + return '' raise RuntimeError(f'grace tool failed for service {spec.service_name()}: {stderr}') + return result.stdout.decode("utf-8") finally: self.mgr.check_mon_command({ diff --git a/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 index 1c10f2b9128..c23bda28196 100644 --- a/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 +++ b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 @@ -43,7 +43,7 @@ NFSv4 { RADOS_KV { UserId = "{{ user }}"; - nodeid = {{ nodeid }}; + nodeid = "{{ nodeid }}"; pool = "{{ pool }}"; namespace = "{{ namespace }}"; } @@ -74,6 +74,12 @@ RGW { name = "client.{{ rgw_user }}"; } +{% if use_old_nodeid %} +Ceph { + Use_old_uuid = true; +} + +{% endif %} {% if tls_add %} TLS_CONFIG{ Enable_TLS = {{ tls_add }}; diff --git a/src/pybind/mgr/cephadm/tests/services/test_ingress.py b/src/pybind/mgr/cephadm/tests/services/test_ingress.py index ae258350003..f2b6328238a 100644 --- a/src/pybind/mgr/cephadm/tests/services/test_ingress.py +++ b/src/pybind/mgr/cephadm/tests/services/test_ingress.py @@ -1210,7 +1210,7 @@ class TestIngressService: '\n' 'RADOS_KV {\n' ' UserId = "nfs.foo.test.0.0";\n' - ' nodeid = 0;\n' + ' nodeid = "0";\n' ' pool = ".nfs";\n' ' namespace = "foo";\n' '}\n' diff --git a/src/pybind/mgr/cephadm/tests/test_migration.py b/src/pybind/mgr/cephadm/tests/test_migration.py index 6e74ad6df76..8b53e01cd56 100644 --- a/src/pybind/mgr/cephadm/tests/test_migration.py +++ b/src/pybind/mgr/cephadm/tests/test_migration.py @@ -216,6 +216,7 @@ def test_migrate_service_id_mds_one(cephadm_module: CephadmOrchestrator): assert len(cephadm_module.spec_store.all_specs) == 0 +@mock.patch("cephadm.services.nfs.NFSService.run_grace_tool", mock.MagicMock()) @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('[]')) def test_migrate_nfs_initial(cephadm_module: CephadmOrchestrator): with with_host(cephadm_module, 'host1'): @@ -249,6 +250,7 @@ def test_migrate_nfs_initial(cephadm_module: CephadmOrchestrator): assert cephadm_module.migration_current == LAST_MIGRATION +@mock.patch("cephadm.services.nfs.NFSService.run_grace_tool", mock.MagicMock()) @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('[]')) def test_migrate_nfs_initial_octopus(cephadm_module: CephadmOrchestrator): with with_host(cephadm_module, 'host1'):