From: Adam King Date: Wed, 20 Jul 2022 21:55:07 +0000 (-0400) Subject: mgr/cephadm: support for nfs backed by VIP X-Git-Tag: v17.2.7~362^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9ad04d3b83fb6b26aafb1d6efad37995550b43b0;p=ceph.git mgr/cephadm: support for nfs backed by VIP Fixes: https://tracker.ceph.com/issues/55663 Signed-off-by: Adam King (cherry picked from commit bb0a0f9d87f50b27c8b32aff8288f14fc3b97045) --- diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 9b708d52bc92..b5518ddb68ce 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1874,7 +1874,9 @@ Then run the following: ) if spec.service_type == 'ingress': # ingress has 2 daemons running per host - sm[nm].size *= 2 + # but only if it's the full ingress service, not for keepalive-only + if not cast(IngressSpec, spec).keepalive_only: + sm[nm].size *= 2 # factor daemons into status for h, dm in self.cache.get_daemons_with_volatile_status(): diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index 0c6a0b208d32..f8b471771463 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -671,8 +671,8 @@ class CephadmServe: else None ), allow_colo=svc.allow_colo(), - primary_daemon_type=svc.primary_daemon_type(), - per_host_daemon_type=svc.per_host_daemon_type(), + primary_daemon_type=svc.primary_daemon_type(spec), + per_host_daemon_type=svc.per_host_daemon_type(spec), rank_map=rank_map, ) diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index cde7a021062f..cac88d8a729d 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -169,13 +169,13 @@ class CephadmService(metaclass=ABCMeta): """ return False - def primary_daemon_type(self) -> str: + def primary_daemon_type(self, spec: Optional[ServiceSpec] = None) -> str: """ This is the type of the primary (usually only) daemon to be deployed. """ return self.TYPE - def per_host_daemon_type(self) -> Optional[str]: + def per_host_daemon_type(self, spec: Optional[ServiceSpec] = None) -> Optional[str]: """ If defined, this type of daemon will be deployed once for each host containing one or more daemons of the primary type. diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index 671660d3f067..65b548f1fccf 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -4,7 +4,7 @@ import random import string from typing import List, Dict, Any, Tuple, cast, Optional -from ceph.deployment.service_spec import IngressSpec +from ceph.deployment.service_spec import ServiceSpec, IngressSpec from mgr_util import build_url from cephadm import utils from orchestrator import OrchestratorError, DaemonDescription @@ -17,10 +17,25 @@ class IngressService(CephService): TYPE = 'ingress' MAX_KEEPALIVED_PASS_LEN = 8 - def primary_daemon_type(self) -> str: + def primary_daemon_type(self, spec: Optional[ServiceSpec] = None) -> str: + if spec: + ispec = cast(IngressSpec, spec) + # in keepalive only setups, we are only deploying keepalived, + # so that should be marked as the primary daemon type. Otherwise, + # we consider haproxy to be the primary. + if hasattr(spec, 'keepalive_only') and ispec.keepalive_only: + return 'keepalived' return 'haproxy' - def per_host_daemon_type(self) -> Optional[str]: + def per_host_daemon_type(self, spec: Optional[ServiceSpec] = None) -> Optional[str]: + if spec: + ispec = cast(IngressSpec, spec) + # if we are using "keepalive_only" mode on this ingress service + # we are only deploying keepalived daemons, so there should + # only be a primary daemon type and the per host daemon type + # should be empty + if hasattr(spec, 'keepalive_only') and ispec.keepalive_only: + return None return 'keepalived' def prepare_create( @@ -216,7 +231,7 @@ class IngressService(CephService): daemons = self.mgr.cache.get_daemons_by_service(spec.service_name()) - if not daemons: + if not daemons and not spec.keepalive_only: raise OrchestratorError( f'Failed to generate keepalived.conf: No daemons deployed for {spec.service_name()}') diff --git a/src/pybind/mgr/cephadm/services/nfs.py b/src/pybind/mgr/cephadm/services/nfs.py index ee53283bd9c1..70e771679c09 100644 --- a/src/pybind/mgr/cephadm/services/nfs.py +++ b/src/pybind/mgr/cephadm/services/nfs.py @@ -92,6 +92,9 @@ class NFSService(CephService): # create the RGW keyring rgw_user = f'{rados_user}-rgw' rgw_keyring = self.create_rgw_keyring(daemon_spec) + bind_addr = spec.virtual_ip if spec.virtual_ip else (daemon_spec.ip if daemon_spec.ip else '') + if not bind_addr: + logger.warning(f'Bind address in {daemon_type}.{daemon_id}\'s ganesha conf is defaulting to empty') # generate the ganesha config def get_ganesha_conf() -> str: @@ -104,7 +107,7 @@ class NFSService(CephService): "url": f'rados://{POOL_NAME}/{spec.service_id}/{spec.rados_config_name()}', # fall back to default NFS port if not present in daemon_spec "port": daemon_spec.ports[0] if daemon_spec.ports else 2049, - "bind_addr": daemon_spec.ip if daemon_spec.ip else '', + "bind_addr": bind_addr, } return self.mgr.template.render('services/nfs/ganesha.conf.j2', context) diff --git a/src/pybind/mgr/nfs/cluster.py b/src/pybind/mgr/nfs/cluster.py index d0adaa49d7de..a104d3e9bf52 100644 --- a/src/pybind/mgr/nfs/cluster.py +++ b/src/pybind/mgr/nfs/cluster.py @@ -9,6 +9,7 @@ from mgr_module import NFS_POOL_NAME as POOL_NAME from ceph.deployment.service_spec import NFSServiceSpec, PlacementSpec, IngressSpec import orchestrator +from orchestrator.module import IngressType from .exception import NFSInvalidOperation, ClusterNotFound from .utils import (available_clusters, restart_nfs_service, conf_obj_name, @@ -53,8 +54,9 @@ class NFSCluster: def _call_orch_apply_nfs( self, cluster_id: str, - placement: Optional[str], + placement: Optional[str] = None, virtual_ip: Optional[str] = None, + ingress_mode: Optional[IngressType] = None, port: Optional[int] = None, ) -> None: if not port: @@ -62,18 +64,27 @@ class NFSCluster: if virtual_ip: # nfs + ingress # run NFS on non-standard port + if not ingress_mode: + ingress_mode = IngressType.default + pspec = PlacementSpec.from_string(placement) + if ingress_mode == IngressType.keepalive_only: + # enforce count=1 for nfs over keepalive only + pspec.count = 1 spec = NFSServiceSpec(service_type='nfs', service_id=cluster_id, - placement=PlacementSpec.from_string(placement), + placement=pspec, # use non-default port so we don't conflict with ingress - port=10000 + port) # semi-arbitrary, fix me someday + port=10000 + port if ingress_mode != IngressType.keepalive_only else port, # semi-arbitrary, fix me someday + virtual_ip=virtual_ip.split('/')[0] if ingress_mode == IngressType.keepalive_only else None) completion = self.mgr.apply_nfs(spec) orchestrator.raise_if_exception(completion) ispec = IngressSpec(service_type='ingress', service_id='nfs.' + cluster_id, backend_service='nfs.' + cluster_id, - frontend_port=port, + placement=pspec, + frontend_port=port if ingress_mode != IngressType.keepalive_only else None, monitor_port=7000 + port, # semi-arbitrary, fix me someday - virtual_ip=virtual_ip) + virtual_ip=virtual_ip, + keepalive_only=(ingress_mode == IngressType.keepalive_only)) completion = self.mgr.apply_ingress(ispec) orchestrator.raise_if_exception(completion) else: @@ -102,6 +113,7 @@ class NFSCluster: placement: Optional[str], virtual_ip: Optional[str], ingress: Optional[bool] = None, + ingress_mode: Optional[IngressType] = None, port: Optional[int] = None, ) -> Tuple[int, str, str]: @@ -115,6 +127,8 @@ class NFSCluster: raise NFSInvalidOperation('virtual_ip can only be provided with ingress enabled') if not virtual_ip and ingress: raise NFSInvalidOperation('ingress currently requires a virtual_ip') + if ingress_mode and not ingress: + raise NFSInvalidOperation('--ingress-mode must be passed along with --ingress') invalid_str = re.search('[^A-Za-z0-9-_.]', cluster_id) if invalid_str: raise NFSInvalidOperation(f"cluster id {cluster_id} is invalid. " @@ -125,7 +139,7 @@ class NFSCluster: self.create_empty_rados_obj(cluster_id) if cluster_id not in available_clusters(self.mgr): - self._call_orch_apply_nfs(cluster_id, placement, virtual_ip, port) + self._call_orch_apply_nfs(cluster_id, placement, virtual_ip, ingress_mode, port) return 0, "NFS Cluster Created Successfully", "" return 0, "", f"{cluster_id} cluster already exists" except Exception as e: diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 340792e81b33..601224ddbde6 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -4,6 +4,7 @@ from typing import Tuple, Optional, List, Dict, Any from mgr_module import MgrModule, CLICommand, Option, CLICheckNonemptyFileInput import orchestrator +from orchestrator.module import IngressType from .export import ExportMgr from .cluster import NFSCluster @@ -111,11 +112,12 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): placement: Optional[str] = None, ingress: Optional[bool] = None, virtual_ip: Optional[str] = None, + ingress_mode: Optional[IngressType] = None, port: Optional[int] = None) -> Tuple[int, str, str]: """Create an NFS Cluster""" return self.nfs.create_nfs_cluster(cluster_id=cluster_id, placement=placement, virtual_ip=virtual_ip, ingress=ingress, - port=port) + ingress_mode=ingress_mode, port=port) @CLICommand('nfs cluster rm', perm='rw') def _cmd_nfs_cluster_rm(self, cluster_id: str) -> Tuple[int, str, str]: diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 3c4274f43e2f..c971a97076c7 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -166,6 +166,11 @@ class DaemonAction(enum.Enum): rotate_key = 'rotate-key' +class IngressType(enum.Enum): + default = 'default' + keepalive_only = 'keepalive-only' + + def to_format(what: Any, format: Format, many: bool, cls: Any) -> Any: def to_json_1(obj: Any) -> Any: if hasattr(obj, 'to_json'): diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 7f5f2fee381b..275ad6a343ba 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -803,6 +803,7 @@ class NFSServiceSpec(ServiceSpec): config: Optional[Dict[str, str]] = None, networks: Optional[List[str]] = None, port: Optional[int] = None, + virtual_ip: Optional[str] = None, extra_container_args: Optional[List[str]] = None, extra_entrypoint_args: Optional[List[str]] = None, custom_configs: Optional[List[CustomConfig]] = None, @@ -815,6 +816,7 @@ class NFSServiceSpec(ServiceSpec): extra_entrypoint_args=extra_entrypoint_args, custom_configs=custom_configs) self.port = port + self.virtual_ip = virtual_ip def get_port_start(self) -> List[int]: if self.port: @@ -1034,6 +1036,7 @@ class IngressSpec(ServiceSpec): virtual_interface_networks: Optional[List[str]] = [], unmanaged: bool = False, ssl: bool = False, + keepalive_only: bool = False, extra_container_args: Optional[List[str]] = None, extra_entrypoint_args: Optional[List[str]] = None, custom_configs: Optional[List[CustomConfig]] = None, @@ -1064,10 +1067,15 @@ class IngressSpec(ServiceSpec): self.virtual_interface_networks = virtual_interface_networks or [] self.unmanaged = unmanaged self.ssl = ssl + self.keepalive_only = keepalive_only def get_port_start(self) -> List[int]: - return [cast(int, self.frontend_port), - cast(int, self.monitor_port)] + ports = [] + if self.frontend_port is not None: + ports.append(cast(int, self.frontend_port)) + if self.monitor_port is not None: + ports.append(cast(int, self.monitor_port)) + return ports def get_virtual_ip(self) -> Optional[str]: return self.virtual_ip @@ -1078,7 +1086,7 @@ class IngressSpec(ServiceSpec): if not self.backend_service: raise SpecValidationError( 'Cannot add ingress: No backend_service specified') - if not self.frontend_port: + if not self.keepalive_only and not self.frontend_port: raise SpecValidationError( 'Cannot add ingress: No frontend_port specified') if not self.monitor_port: