From 5173b418f7f97c6471f42324b6433fc62c09da35 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 9 Apr 2021 12:34:20 -0400 Subject: [PATCH] mgr/cephadm: HA_RGW -> Ingress This is mostly a rename, with some simplification and cleanup. Signed-off-by: Sage Weil (cherry picked from commit 0894773e95bd0d10ba85768ba8b9116fcd375f94) --- src/pybind/mgr/cephadm/module.py | 54 +++-- src/pybind/mgr/cephadm/serve.py | 60 ++---- .../mgr/cephadm/services/cephadmservice.py | 18 +- src/pybind/mgr/cephadm/services/ha_rgw.py | 132 ------------ src/pybind/mgr/cephadm/services/ingress.py | 199 ++++++++++++++++++ .../templates/services/ingress/haproxy.cfg.j2 | 62 ++++++ .../services/ingress/keepalived.conf.j2 | 32 +++ .../mgr/cephadm/tests/test_scheduling.py | 24 ++- src/pybind/mgr/cephadm/tests/test_spec.py | 33 +-- src/pybind/mgr/orchestrator/_interface.py | 16 +- .../ceph/deployment/service_spec.py | 116 ++++------ 11 files changed, 445 insertions(+), 301 deletions(-) delete mode 100644 src/pybind/mgr/cephadm/services/ha_rgw.py create mode 100644 src/pybind/mgr/cephadm/services/ingress.py create mode 100644 src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 create mode 100644 src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index ecd7118f406af..7060dbcd87f5f 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -25,7 +25,7 @@ from ceph.deployment import inventory from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.service_spec import \ NFSServiceSpec, ServiceSpec, PlacementSpec, assert_valid_host, \ - HostPlacementSpec + HostPlacementSpec, IngressSpec from ceph.utils import str_to_datetime, datetime_to_str, datetime_now from cephadm.serve import CephadmServe from cephadm.services.cephadmservice import CephadmDaemonDeploySpec @@ -47,9 +47,9 @@ from . import utils from .migrations import Migrations from .services.cephadmservice import MonService, MgrService, MdsService, RgwService, \ RbdMirrorService, CrashService, CephadmService, CephfsMirrorService +from .services.ingress import IngressService from .services.container import CustomContainerService from .services.iscsi import IscsiService -from .services.ha_rgw import HA_RGWService from .services.nfs import NFSService from .services.osd import OSDRemovalQueue, OSDService, OSD, NotFoundError from .services.monitoring import GrafanaService, AlertmanagerService, PrometheusService, \ @@ -421,7 +421,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, OSDService, NFSService, MonService, MgrService, MdsService, RgwService, RbdMirrorService, GrafanaService, AlertmanagerService, PrometheusService, NodeExporterService, CrashService, IscsiService, - HA_RGWService, CustomContainerService, CephadmExporter, CephfsMirrorService + IngressService, CustomContainerService, CephadmExporter, CephfsMirrorService ] # https://github.com/python/mypy/issues/8993 @@ -1581,8 +1581,8 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, if service_type == 'nfs': spec = cast(NFSServiceSpec, spec) sm[nm].rados_config_location = spec.rados_config_location() - if spec.service_type == 'ha-rgw': - # ha-rgw has 2 daemons running per host + if spec.service_type == 'ingress': + # ingress has 2 daemons running per host sm[nm].size *= 2 # factor daemons into status @@ -1952,16 +1952,38 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, previews_for_specs.update({host: osd_reports}) return previews_for_specs - def _calc_daemon_deps(self, daemon_type: str, daemon_id: str) -> List[str]: - need = { - 'prometheus': ['mgr', 'alertmanager', 'node-exporter'], - 'grafana': ['prometheus'], - 'alertmanager': ['mgr', 'alertmanager'], - } + def _calc_daemon_deps(self, + spec: Optional[ServiceSpec], + daemon_type: str, + daemon_id: str) -> List[str]: deps = [] - for dep_type in need.get(daemon_type, []): - for dd in self.cache.get_daemons_by_service(dep_type): - deps.append(dd.name()) + if daemon_type == 'haproxy': + # because cephadm creates new daemon instances whenever + # port or ip changes, identifying daemons by name is + # sufficient to detect changes. + if not spec: + return [] + ingress_spec = cast(IngressSpec, spec) + assert ingress_spec.backend_service + daemons = self.cache.get_daemons_by_service(ingress_spec.backend_service) + deps = [d.name() for d in daemons] + elif daemon_type == 'keepalived': + # because cephadm creates new daemon instances whenever + # port or ip changes, identifying daemons by name is + # sufficient to detect changes. + if not spec: + return [] + daemons = self.cache.get_daemons_by_service(spec.service_name()) + deps = [d.name() for d in daemons if d.daemon_type == 'haproxy'] + else: + need = { + 'prometheus': ['mgr', 'alertmanager', 'node-exporter'], + 'grafana': ['prometheus'], + 'alertmanager': ['mgr', 'alertmanager'], + } + for dep_type in need.get(daemon_type, []): + for dd in self.cache.get_daemons_by_service(dep_type): + deps.append(dd.name()) return sorted(deps) @forall_hosts @@ -2105,7 +2127,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, 'mgr': PlacementSpec(count=2), 'mds': PlacementSpec(count=2), 'rgw': PlacementSpec(count=2), - 'ha-rgw': PlacementSpec(count=2), + 'ingress': PlacementSpec(count=2), 'iscsi': PlacementSpec(count=1), 'rbd-mirror': PlacementSpec(count=2), 'cephfs-mirror': PlacementSpec(count=1), @@ -2168,7 +2190,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, return self._apply(spec) @handle_orch_error - def apply_ha_rgw(self, spec: ServiceSpec) -> str: + def apply_ingress(self, spec: ServiceSpec) -> str: return self._apply(spec) @handle_orch_error diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index aaa753f25c935..abd8d324d3597 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -14,14 +14,14 @@ except ImportError: from ceph.deployment import inventory from ceph.deployment.drive_group import DriveGroupSpec -from ceph.deployment.service_spec import ServiceSpec, HA_RGWSpec, CustomContainerSpec +from ceph.deployment.service_spec import ServiceSpec, IngressSpec, CustomContainerSpec from ceph.utils import str_to_datetime, datetime_now import orchestrator from orchestrator import OrchestratorError, set_exception_subject, OrchestratorEvent, \ DaemonDescriptionStatus, daemon_type_to_service, service_to_daemon_types from cephadm.services.cephadmservice import CephadmDaemonDeploySpec -from cephadm.schedule import HostAssignment, DaemonPlacement +from cephadm.schedule import HostAssignment from cephadm.utils import forall_hosts, cephadmNoImage, is_repo_digest, \ CephadmNoImage, CEPH_TYPES, ContainerInspectInfo from mgr_module import MonCommandFailed @@ -559,8 +559,11 @@ class CephadmServe: hosts=self.mgr._hosts_with_daemon_inventory(), daemons=daemons, networks=self.mgr.cache.networks, - filter_new_host=matches_network if service_type == 'mon' - else virtual_ip_allowed if service_type == 'ha-rgw' else None, + filter_new_host=( + matches_network if service_type == 'mon' + else virtual_ip_allowed if service_type == 'ingress' + else None + ), allow_colo=svc.allow_colo(), ) @@ -587,19 +590,23 @@ class CephadmServe: self.log.debug('Hosts that will receive new daemons: %s' % slots_to_add) self.log.debug('Daemons that will be removed: %s' % daemons_to_remove) - if service_type == 'ha-rgw': - spec = self.update_ha_rgw_definitive_hosts(spec, all_slots, slots_to_add) - for slot in slots_to_add: for daemon_type in service_to_daemon_types(service_type): + if daemon_type != 'keepalived': + slot_ports = slot.ports + slot_ip = slot.ip + else: + slot_ports = [] + slot_ip = None + # first remove daemon on conflicting port? - if slot.ports: + if slot_ports: for d in daemons_to_remove: if d.hostname != slot.hostname: continue - if not (set(d.ports or []) & set(slot.ports)): + if not (set(d.ports or []) & set(slot_ports)): continue - if d.ip and slot.ip and d.ip != slot.ip: + if d.ip and slot_ip and d.ip != slot_ip: continue self.log.info( f'Removing {d.name()} before deploying to {slot} to avoid a port conflict' @@ -624,8 +631,8 @@ class CephadmServe: daemon_spec = svc.make_daemon_spec( slot.hostname, daemon_id, slot.network, spec, daemon_type=daemon_type, - ports=slot.ports, - ip=slot.ip, + ports=slot_ports, + ip=slot_ip, ) self.log.debug('Placing %s.%s on host %s' % ( daemon_type, daemon_id, slot.hostname)) @@ -702,7 +709,7 @@ class CephadmServe: else: dd.is_active = False - deps = self.mgr._calc_daemon_deps(dd.daemon_type, dd.daemon_id) + deps = self.mgr._calc_daemon_deps(spec, dd.daemon_type, dd.daemon_id) last_deps, last_config = self.mgr.cache.get_daemon_last_config_deps( dd.hostname, dd.name()) if last_deps is None: @@ -788,29 +795,6 @@ class CephadmServe: # FIXME: we assume the first digest here is the best self.mgr.set_container_image(entity, image_info.repo_digests[0]) - # ha-rgw needs definitve host list to create keepalived config files - # if definitive host list has changed, all ha-rgw daemons must get new - # config, including those that are already on the correct host and not - # going to be deployed - def update_ha_rgw_definitive_hosts( - self, - spec: ServiceSpec, - hosts: List[DaemonPlacement], - add_hosts: List[DaemonPlacement] - ) -> HA_RGWSpec: - spec = cast(HA_RGWSpec, spec) - hostnames = [p.hostname for p in hosts] - add_hostnames = [p.hostname for p in add_hosts] - if not (set(hostnames) == set(spec.definitive_host_list)): - spec.definitive_host_list = hostnames - ha_rgw_daemons = self.mgr.cache.get_daemons_by_service(spec.service_name()) - for daemon in ha_rgw_daemons: - if daemon.hostname in hostnames and daemon.hostname not in add_hostnames: - assert daemon.hostname is not None - self.mgr.cache.schedule_daemon_action( - daemon.hostname, daemon.name(), 'reconfig') - return spec - def _create_daemon(self, daemon_spec: CephadmDaemonDeploySpec, reconfig: bool = False, @@ -841,12 +825,12 @@ class CephadmServe: self._deploy_cephadm_binary(daemon_spec.host) if daemon_spec.daemon_type == 'haproxy': - haspec = cast(HA_RGWSpec, self.mgr.spec_store[daemon_spec.service_name].spec) + haspec = cast(IngressSpec, self.mgr.spec_store[daemon_spec.service_name].spec) if haspec.haproxy_container_image: image = haspec.haproxy_container_image if daemon_spec.daemon_type == 'keepalived': - haspec = cast(HA_RGWSpec, self.mgr.spec_store[daemon_spec.service_name].spec) + haspec = cast(IngressSpec, self.mgr.spec_store[daemon_spec.service_name].spec) if haspec.keepalived_container_image: image = haspec.keepalived_container_image diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index e777a0e80dacd..f3a4dd497242a 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -1,7 +1,7 @@ import errno import json -import re import logging +import re from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING, List, Callable, TypeVar, \ Optional, Dict, Any, Tuple, NewType, cast @@ -394,7 +394,7 @@ class CephService(CephadmService): """ # despite this mapping entity names to daemons, self.TYPE within # the CephService class refers to service types, not daemon types - if self.TYPE in ['rgw', 'rbd-mirror', 'cephfs-mirror', 'nfs', "iscsi", 'ha-rgw']: + if self.TYPE in ['rgw', 'rbd-mirror', 'cephfs-mirror', 'nfs', "iscsi", 'ingress']: return AuthEntity(f'client.{self.TYPE}.{daemon_id}') elif self.TYPE == 'crash': if host == "": @@ -832,15 +832,15 @@ class RgwService(CephService): force: bool = False, known: Optional[List[str]] = None # output argument ) -> HandleCommandResult: - # if load balancer (ha-rgw) is present block if only 1 daemon up otherwise ok + # if load balancer (ingress) is present block if only 1 daemon up otherwise ok # if no load balancer, warn if > 1 daemon, block if only 1 daemon - def ha_rgw_present() -> bool: - running_ha_rgw_daemons = [ - daemon for daemon in self.mgr.cache.get_daemons_by_type('ha-rgw') if daemon.status == 1] + def ingress_present() -> bool: + running_ingress_daemons = [ + daemon for daemon in self.mgr.cache.get_daemons_by_type('ingress') if daemon.status == 1] running_haproxy_daemons = [ - daemon for daemon in running_ha_rgw_daemons if daemon.daemon_type == 'haproxy'] + daemon for daemon in running_ingress_daemons if daemon.daemon_type == 'haproxy'] running_keepalived_daemons = [ - daemon for daemon in running_ha_rgw_daemons if daemon.daemon_type == 'keepalived'] + daemon for daemon in running_ingress_daemons if daemon.daemon_type == 'keepalived'] # check that there is at least one haproxy and keepalived daemon running if running_haproxy_daemons and running_keepalived_daemons: return True @@ -853,7 +853,7 @@ class RgwService(CephService): # if reached here, there is > 1 rgw daemon. # Say okay if load balancer present or force flag set - if ha_rgw_present() or force: + if ingress_present() or force: return HandleCommandResult(0, warn_message, '') # if reached here, > 1 RGW daemon, no load balancer and no force flag. diff --git a/src/pybind/mgr/cephadm/services/ha_rgw.py b/src/pybind/mgr/cephadm/services/ha_rgw.py deleted file mode 100644 index 6f7abad40c52e..0000000000000 --- a/src/pybind/mgr/cephadm/services/ha_rgw.py +++ /dev/null @@ -1,132 +0,0 @@ -import logging -from typing import List, cast, Tuple, Dict, Any - -from ceph.deployment.service_spec import HA_RGWSpec - -from .cephadmservice import CephadmDaemonDeploySpec, CephService -from ..utils import resolve_ip - -logger = logging.getLogger(__name__) - - -class HA_RGWService(CephService): - TYPE = 'ha-rgw' - - class rgw_server(): - def __init__(self, hostname: str, address: str, port: int): - self.name = hostname - self.ip = address - self.port = port - - def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec: - assert daemon_spec.daemon_type == 'haproxy' or daemon_spec.daemon_type == 'keepalived' - if daemon_spec.daemon_type == 'haproxy': - return self.haproxy_prepare_create(daemon_spec) - else: - return self.keepalived_prepare_create(daemon_spec) - - def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]: - assert daemon_spec.daemon_type == 'haproxy' or daemon_spec.daemon_type == 'keepalived' - - if daemon_spec.daemon_type == 'haproxy': - return self.haproxy_generate_config(daemon_spec) - else: - return self.keepalived_generate_config(daemon_spec) - - def haproxy_prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec: - assert daemon_spec.daemon_type == 'haproxy' - - daemon_id = daemon_spec.daemon_id - host = daemon_spec.host - spec = cast(HA_RGWSpec, self.mgr.spec_store[daemon_spec.service_name].spec) - - logger.info('Create daemon %s on host %s with spec %s' % ( - daemon_id, host, spec)) - - daemon_spec.final_config, daemon_spec.deps = self.haproxy_generate_config(daemon_spec) - - return daemon_spec - - def keepalived_prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec: - assert daemon_spec.daemon_type == 'keepalived' - - daemon_id = daemon_spec.daemon_id - host = daemon_spec.host - spec = cast(HA_RGWSpec, self.mgr.spec_store[daemon_spec.service_name].spec) - - logger.info('Create daemon %s on host %s with spec %s' % ( - daemon_id, host, spec)) - - daemon_spec.final_config, daemon_spec.deps = self.keepalived_generate_config(daemon_spec) - - return daemon_spec - - def haproxy_generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]: - spec = cast(HA_RGWSpec, self.mgr.spec_store[daemon_spec.service_name].spec) - - rgw_daemons = self.mgr.cache.get_daemons_by_type('rgw') - rgw_servers = [] - for daemon in rgw_daemons: - assert daemon.hostname is not None - rgw_servers.append(self.rgw_server( - daemon.name(), - resolve_ip(daemon.hostname), - daemon.ports[0] if daemon.ports else 80 - )) - - # virtual ip address cannot have netmask attached when passed to haproxy config - # since the port is added to the end and something like 123.123.123.10/24:8080 is invalid - virtual_ip_address = spec.virtual_ip_address - if "/" in str(spec.virtual_ip_address): - just_ip = str(spec.virtual_ip_address).split('/')[0] - virtual_ip_address = just_ip - - ha_context = {'spec': spec, 'rgw_servers': rgw_servers, - 'virtual_ip_address': virtual_ip_address} - - haproxy_conf = self.mgr.template.render('services/haproxy/haproxy.cfg.j2', ha_context) - - config_files = { - 'files': { - "haproxy.cfg": haproxy_conf, - } - } - if spec.ha_proxy_frontend_ssl_certificate: - ssl_cert = spec.ha_proxy_frontend_ssl_certificate - if isinstance(ssl_cert, list): - ssl_cert = '\n'.join(ssl_cert) - config_files['files']['haproxy.pem'] = ssl_cert - - return config_files, [] - - def keepalived_generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]: - host = daemon_spec.host - - spec = cast(HA_RGWSpec, self.mgr.spec_store[daemon_spec.service_name].spec) - - all_hosts = spec.definitive_host_list - - # set state. first host in placement is master all others backups - state = 'BACKUP' - if all_hosts[0] == host: - state = 'MASTER' - - # remove host, daemon is being deployed on from all_hosts list for - # other_ips in conf file and converter to ips - all_hosts.remove(host) - other_ips = [resolve_ip(h) for h in all_hosts] - - ka_context = {'spec': spec, 'state': state, - 'other_ips': other_ips, - 'host_ip': resolve_ip(host)} - - keepalived_conf = self.mgr.template.render( - 'services/keepalived/keepalived.conf.j2', ka_context) - - config_file = { - 'files': { - "keepalived.conf": keepalived_conf, - } - } - - return config_file, [] diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py new file mode 100644 index 0000000000000..b1c43e70b31d7 --- /dev/null +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -0,0 +1,199 @@ +import ipaddress +import logging +import random +import string +from typing import List, Dict, Any, Tuple, cast + +from ceph.deployment.service_spec import IngressSpec +from cephadm.utils import resolve_ip + +from cephadm.services.cephadmservice import CephadmDaemonDeploySpec, CephService + +logger = logging.getLogger(__name__) + + +class IngressService(CephService): + TYPE = 'ingress' + + def prepare_create( + self, + daemon_spec: CephadmDaemonDeploySpec, + ) -> CephadmDaemonDeploySpec: + if daemon_spec.daemon_type == 'haproxy': + return self.haproxy_prepare_create(daemon_spec) + if daemon_spec.daemon_type == 'keepalived': + return self.keepalived_prepare_create(daemon_spec) + assert False, "unexpected daemon type" + + def generate_config( + self, + daemon_spec: CephadmDaemonDeploySpec + ) -> Tuple[Dict[str, Any], List[str]]: + if daemon_spec.daemon_type == 'haproxy': + return self.haproxy_generate_config(daemon_spec) + else: + return self.keepalived_generate_config(daemon_spec) + assert False, "unexpected daemon type" + + def haproxy_prepare_create( + self, + daemon_spec: CephadmDaemonDeploySpec, + ) -> CephadmDaemonDeploySpec: + assert daemon_spec.daemon_type == 'haproxy' + + daemon_id = daemon_spec.daemon_id + host = daemon_spec.host + spec = cast(IngressSpec, self.mgr.spec_store[daemon_spec.service_name].spec) + + logger.info('Create daemon haproxy.%s on host %s with spec %s' % ( + daemon_id, host, spec)) + + daemon_spec.final_config, daemon_spec.deps = self.haproxy_generate_config(daemon_spec) + + return daemon_spec + + def haproxy_generate_config( + self, + daemon_spec: CephadmDaemonDeploySpec, + ) -> Tuple[Dict[str, Any], List[str]]: + spec = cast(IngressSpec, self.mgr.spec_store[daemon_spec.service_name].spec) + assert spec.backend_service + daemons = self.mgr.cache.get_daemons_by_service(spec.backend_service) + deps = [d.name() for d in daemons] + + # generate password? + pw_key = f'{spec.service_name()}/monitor_password' + password = self.mgr.get_store(pw_key) + if password is None: + if not spec.monitor_password: + password = ''.join(random.choice(string.ascii_lowercase) for _ in range(20)) + self.mgr.set_store(pw_key, password) + else: + if spec.monitor_password: + self.mgr.set_store(pw_key, None) + if spec.monitor_password: + password = spec.monitor_password + + haproxy_conf = self.mgr.template.render( + 'services/ingress/haproxy.cfg.j2', + { + 'spec': spec, + 'servers': [ + { + 'name': d.name(), + 'ip': d.ip or resolve_ip(str(d.hostname)), + 'port': d.ports[0], + } for d in daemons if d.ports + ], + 'user': spec.monitor_user or 'admin', + 'password': password, + 'ip': daemon_spec.ip or '*', + 'frontend_port': daemon_spec.ports[0] if daemon_spec.ports else spec.frontend_port, + 'monitor_port': daemon_spec.ports[1] if daemon_spec.ports else spec.monitor_port, + } + ) + config_files = { + 'files': { + "haproxy.cfg": haproxy_conf, + } + } + if spec.ssl_cert: + ssl_cert = spec.ssl_cert + if isinstance(ssl_cert, list): + ssl_cert = '\n'.join(ssl_cert) + config_files['files']['haproxy.pem'] = ssl_cert + + return config_files, sorted(deps) + + def keepalived_prepare_create( + self, + daemon_spec: CephadmDaemonDeploySpec, + ) -> CephadmDaemonDeploySpec: + assert daemon_spec.daemon_type == 'keepalived' + + daemon_id = daemon_spec.daemon_id + host = daemon_spec.host + spec = cast(IngressSpec, self.mgr.spec_store[daemon_spec.service_name].spec) + + logger.info('Create daemon keepalived.%s on host %s with spec %s' % ( + daemon_id, host, spec)) + + daemon_spec.final_config, daemon_spec.deps = self.keepalived_generate_config(daemon_spec) + + return daemon_spec + + def keepalived_generate_config( + self, + daemon_spec: CephadmDaemonDeploySpec, + ) -> Tuple[Dict[str, Any], List[str]]: + spec = cast(IngressSpec, self.mgr.spec_store[daemon_spec.service_name].spec) + assert spec.backend_service + + # generate password? + pw_key = f'{spec.service_name()}/keepalived_password' + password = self.mgr.get_store(pw_key) + if password is None: + if not spec.keepalived_password: + password = ''.join(random.choice(string.ascii_lowercase) for _ in range(20)) + self.mgr.set_store(pw_key, password) + else: + if spec.keepalived_password: + self.mgr.set_store(pw_key, None) + if spec.keepalived_password: + password = spec.keepalived_password + + daemons = self.mgr.cache.get_daemons_by_service(spec.service_name()) + deps = sorted([d.name() for d in daemons if d.daemon_type == 'haproxy']) + + host = daemon_spec.host + hosts = sorted(list(set([str(d.hostname) for d in daemons]))) + + # interface + interface = 'eth0' + for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): + logger.info(f'subnet {subnet} ifaces {ifaces} virtual_ip {spec.virtual_ip}') + if ifaces and ipaddress.ip_address(spec.virtual_ip) in ipaddress.ip_network(subnet): + logger.info(f'{spec.virtual_ip} is in {subnet}') + interface = list(ifaces.keys())[0] + break + + # script to monitor health + script = '/usr/bin/false' + for d in daemons: + if d.hostname == host: + if d.daemon_type == 'haproxy': + assert d.ports + port = d.ports[1] # monitoring port + script = f'/usr/bin/curl http://{d.ip or "localhost"}:{port}/health' + assert script + + # set state. first host in placement is master all others backups + state = 'BACKUP' + if hosts[0] == host: + state = 'MASTER' + + # remove host, daemon is being deployed on from hosts list for + # other_ips in conf file and converter to ips + hosts.remove(host) + other_ips = [resolve_ip(h) for h in hosts] + + keepalived_conf = self.mgr.template.render( + 'services/ingress/keepalived.conf.j2', + { + 'spec': spec, + 'script': script, + 'password': password, + 'interface': interface, + 'state': state, + 'other_ips': other_ips, + 'host_ip': resolve_ip(host), + } + ) + + config_file = { + 'files': { + "keepalived.conf": keepalived_conf, + } + } + + return config_file, deps diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 new file mode 100644 index 0000000000000..755067a2ca056 --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 @@ -0,0 +1,62 @@ +# {{ cephadm_managed }} +global + log 127.0.0.1 local2 + chroot /var/lib/haproxy + pidfile /var/lib/haproxy/haproxy.pid + maxconn 8000 + daemon + stats socket /var/lib/haproxy/stats +{% if spec.ssl_cert %} + {% if spec.ssl_dh_param %} + tune.ssl.default-dh-param {{ spec.ssl_dh_param }} + {% endif %} + {% if spec.ssl_ciphers %} + ssl-default-bind-ciphers {{ spec.ssl_ciphers | join(':') }} + {% endif %} + {% if spec.ssl_options %} + ssl-default-bind-options {{ spec.ssl_options | join(' ') }} + {% endif %} +{% endif %} + +defaults + mode http + log global + option httplog + option dontlognull + option http-server-close + option forwardfor except 127.0.0.0/8 + option redispatch + retries 3 + timeout http-request 1s + timeout queue 20s + timeout connect 5s + timeout client 1s + timeout server 1s + timeout http-keep-alive 5s + timeout check 5s + maxconn 8000 + +frontend stats + bind {{ ip }}:{{ monitor_port }} + stats enable + stats uri /stats + stats refresh 10s + stats auth {{ user }}:{{ password }} + http-request use-service prometheus-exporter if { path /metrics } + monitor-uri /health + +frontend frontend +{% if spec.ssl_cert %} + bind {{ ip }}:{{ frontend_port }} ssl crt /var/lib/haproxy/haproxy.pem +{% else %} + bind {{ ip }}:{{ frontend_port }} +{% endif %} + default_backend backend + +backend backend + option forwardfor + balance static-rr + option httpchk HEAD / HTTP/1.0 + {% for server in servers %} + server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 + {% endfor %} diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 new file mode 100644 index 0000000000000..2c4e326f6db1f --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 @@ -0,0 +1,32 @@ +# {{ cephadm_managed }} +vrrp_script check_backend { + script "{{ script }}" + weight -20 + interval 2 + rise 2 + fall 2 +} + +vrrp_instance VI_0 { + state {{ state }} + priority 100 + interface {{ interface }} + virtual_router_id 51 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{ password }} + } + unicast_src_ip {{ host_ip }} + unicast_peer { + {% for ip in other_ips %} + {{ ip }} + {% endfor %} + } + virtual_ipaddress { + {{ spec.virtual_ip }} dev {{ interface }} + } + track_script { + check_backend + } +} diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py index bed12651f9437..7a135b3a196d0 100644 --- a/src/pybind/mgr/cephadm/tests/test_scheduling.py +++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py @@ -6,7 +6,7 @@ from typing import NamedTuple, List, Dict import pytest from ceph.deployment.hostspec import HostSpec -from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, ServiceSpecValidationError +from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, ServiceSpecValidationError, IngressSpec from cephadm.module import HostAssignment from cephadm.schedule import DaemonPlacement @@ -810,6 +810,27 @@ class NodeAssignmentTest4(NamedTuple): 'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'], [] ), + NodeAssignmentTest4( + IngressSpec( + service_type='ingress', + service_id='rgw.foo', + frontend_port=443, + monitor_port=8888, + virtual_ip='10.0.0.20/8', + backend_service='rgw.foo', + placement=PlacementSpec(label='foo'), + networks=['10.0.0.0/8'], + ), + { + 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}}, + 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}}, + 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}}, + }, + [], + ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)'], + ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)'], + [] + ), ]) def test_node_assignment4(spec, networks, daemons, expected, expected_add, expected_remove): @@ -819,6 +840,7 @@ def test_node_assignment4(spec, networks, daemons, daemons=daemons, allow_colo=True, networks=networks, + primary_daemon_type='haproxy' if spec.service_type == 'ingress' else spec.service_type, ).place() got = [str(p) for p in all_slots] diff --git a/src/pybind/mgr/cephadm/tests/test_spec.py b/src/pybind/mgr/cephadm/tests/test_spec.py index 8eb8e8b7b9d55..d46cbfe7f17ff 100644 --- a/src/pybind/mgr/cephadm/tests/test_spec.py +++ b/src/pybind/mgr/cephadm/tests/test_spec.py @@ -668,37 +668,24 @@ def test_custom_container_spec_config_json(): assert key not in config_json -def test_HA_RGW_spec(): - yaml_str = """service_type: ha-rgw -service_id: haproxy_for_rgw +def test_ingress_spec(): + yaml_str = """service_type: ingress +service_id: rgw.foo placement: hosts: - host1 - host2 - host3 spec: - virtual_ip_interface: eth0 - virtual_ip_address: 192.168.20.1/24 + virtual_ip: 192.168.20.1/24 + backend_service: rgw.foo frontend_port: 8080 - ha_proxy_port: 1967 - ha_proxy_stats_enabled: true - ha_proxy_stats_user: admin - ha_proxy_stats_password: admin - ha_proxy_enable_prometheus_exporter: true - ha_proxy_monitor_uri: /haproxy_health - keepalived_password: admin + monitor_port: 8081 """ yaml_file = yaml.safe_load(yaml_str) spec = ServiceSpec.from_json(yaml_file) - assert spec.service_type == "ha-rgw" - assert spec.service_id == "haproxy_for_rgw" - assert spec.virtual_ip_interface == "eth0" - assert spec.virtual_ip_address == "192.168.20.1/24" + assert spec.service_type == "ingress" + assert spec.service_id == "rgw.foo" + assert spec.virtual_ip == "192.168.20.1/24" assert spec.frontend_port == 8080 - assert spec.ha_proxy_port == 1967 - assert spec.ha_proxy_stats_enabled is True - assert spec.ha_proxy_stats_user == "admin" - assert spec.ha_proxy_stats_password == "admin" - assert spec.ha_proxy_enable_prometheus_exporter is True - assert spec.ha_proxy_monitor_uri == "/haproxy_health" - assert spec.keepalived_password == "admin" + assert spec.monitor_port == 8081 diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index adb2eb12671c5..9d1808fda9390 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -31,7 +31,7 @@ import yaml from ceph.deployment import inventory from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, \ - ServiceSpecValidationError, IscsiServiceSpec, HA_RGWSpec + ServiceSpecValidationError, IscsiServiceSpec, IngressSpec from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.hostspec import HostSpec from ceph.utils import datetime_to_str, str_to_datetime @@ -450,7 +450,7 @@ class Orchestrator(object): 'prometheus': self.apply_prometheus, 'rbd-mirror': self.apply_rbd_mirror, 'rgw': self.apply_rgw, - 'ha-rgw': self.apply_ha_rgw, + 'ingress': self.apply_ingress, 'host': self.add_host, 'cephadm-exporter': self.apply_cephadm_exporter, } @@ -596,8 +596,8 @@ class Orchestrator(object): """Update RGW cluster""" raise NotImplementedError() - def apply_ha_rgw(self, spec: HA_RGWSpec) -> OrchResult[str]: - """Update ha-rgw daemons""" + def apply_ingress(self, spec: IngressSpec) -> OrchResult[str]: + """Update ingress daemons""" raise NotImplementedError() def apply_rbd_mirror(self, spec: ServiceSpec) -> OrchResult[str]: @@ -687,8 +687,8 @@ def daemon_type_to_service(dtype: str) -> str: 'mds': 'mds', 'rgw': 'rgw', 'osd': 'osd', - 'haproxy': 'ha-rgw', - 'keepalived': 'ha-rgw', + 'haproxy': 'ingress', + 'keepalived': 'ingress', 'iscsi': 'iscsi', 'rbd-mirror': 'rbd-mirror', 'cephfs-mirror': 'cephfs-mirror', @@ -712,7 +712,7 @@ def service_to_daemon_types(stype: str) -> List[str]: 'mds': ['mds'], 'rgw': ['rgw'], 'osd': ['osd'], - 'ha-rgw': ['haproxy', 'keepalived'], + 'ingress': ['haproxy', 'keepalived'], 'iscsi': ['iscsi'], 'rbd-mirror': ['rbd-mirror'], 'cephfs-mirror': ['cephfs-mirror'], @@ -810,8 +810,6 @@ class DaemonDescription(object): # The type of service (osd, mon, mgr, etc.) self.daemon_type = daemon_type - assert daemon_type not in ['HA_RGW', 'ha-rgw'] - # The orchestrator will have picked some names for daemons, # typically either based on hostnames or on pod names. # This is the in mds., the ID that will appear diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index c6e607f171eed..1360017a3c9bb 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -428,8 +428,8 @@ class ServiceSpec(object): """ KNOWN_SERVICE_TYPES = 'alertmanager crash grafana iscsi mds mgr mon nfs ' \ 'node-exporter osd prometheus rbd-mirror rgw ' \ - 'container cephadm-exporter ha-rgw cephfs-mirror'.split() - REQUIRES_SERVICE_ID = 'iscsi mds nfs osd rgw container ha-rgw '.split() + 'container cephadm-exporter ingress cephfs-mirror'.split() + REQUIRES_SERVICE_ID = 'iscsi mds nfs osd rgw container ingress '.split() MANAGED_CONFIG_OPTIONS = [ 'mds_join_fs', ] @@ -444,7 +444,7 @@ class ServiceSpec(object): 'osd': DriveGroupSpec, 'iscsi': IscsiServiceSpec, 'alertmanager': AlertManagerSpec, - 'ha-rgw': HA_RGWSpec, + 'ingress': IngressSpec, 'container': CustomContainerSpec, }.get(service_type, cls) if ret == ServiceSpec and not service_type: @@ -851,97 +851,67 @@ class AlertManagerSpec(ServiceSpec): yaml.add_representer(AlertManagerSpec, ServiceSpec.yaml_representer) -class HA_RGWSpec(ServiceSpec): +class IngressSpec(ServiceSpec): def __init__(self, - service_type: str = 'ha-rgw', + service_type: str = 'ingress', service_id: Optional[str] = None, config: Optional[Dict[str, str]] = None, networks: Optional[List[str]] = None, placement: Optional[PlacementSpec] = None, - virtual_ip_interface: Optional[str] = None, - virtual_ip_address: Optional[str] = None, + backend_service: Optional[str] = None, frontend_port: Optional[int] = None, - ha_proxy_port: Optional[int] = None, - ha_proxy_stats_enabled: Optional[bool] = None, - ha_proxy_stats_user: Optional[str] = None, - ha_proxy_stats_password: Optional[str] = None, - ha_proxy_enable_prometheus_exporter: Optional[bool] = None, - ha_proxy_monitor_uri: Optional[str] = None, + ssl_cert: Optional[str] = None, + ssl_dh_param: Optional[str] = None, + ssl_ciphers: Optional[List[str]] = None, + ssl_options: Optional[List[str]] = None, + monitor_port: Optional[int] = None, + monitor_user: Optional[str] = None, + monitor_password: Optional[str] = None, + enable_stats: Optional[bool] = None, keepalived_password: Optional[str] = None, - ha_proxy_frontend_ssl_certificate: Optional[str] = None, - ha_proxy_frontend_ssl_port: Optional[int] = None, - ha_proxy_ssl_dh_param: Optional[str] = None, - ha_proxy_ssl_ciphers: Optional[List[str]] = None, - ha_proxy_ssl_options: Optional[List[str]] = None, + virtual_ip: Optional[str] = None, haproxy_container_image: Optional[str] = None, keepalived_container_image: Optional[str] = None, - definitive_host_list: Optional[List[str]] = None ): - assert service_type == 'ha-rgw' - super(HA_RGWSpec, self).__init__('ha-rgw', service_id=service_id, - placement=placement, config=config, - networks=networks) - - self.virtual_ip_interface = virtual_ip_interface - self.virtual_ip_address = virtual_ip_address + assert service_type == 'ingress' + super(IngressSpec, self).__init__( + 'ingress', service_id=service_id, + placement=placement, config=config, + networks=networks + ) + self.backend_service = backend_service self.frontend_port = frontend_port - self.ha_proxy_port = ha_proxy_port - self.ha_proxy_stats_enabled = ha_proxy_stats_enabled - self.ha_proxy_stats_user = ha_proxy_stats_user - self.ha_proxy_stats_password = ha_proxy_stats_password - self.ha_proxy_enable_prometheus_exporter = ha_proxy_enable_prometheus_exporter - self.ha_proxy_monitor_uri = ha_proxy_monitor_uri + self.ssl_cert = ssl_cert + self.ssl_dh_param = ssl_dh_param + self.ssl_ciphers = ssl_ciphers + self.ssl_options = ssl_options + self.monitor_port = monitor_port + self.monitor_user = monitor_user + self.monitor_password = monitor_password self.keepalived_password = keepalived_password - self.ha_proxy_frontend_ssl_certificate = ha_proxy_frontend_ssl_certificate - self.ha_proxy_frontend_ssl_port = ha_proxy_frontend_ssl_port - self.ha_proxy_ssl_dh_param = ha_proxy_ssl_dh_param - self.ha_proxy_ssl_ciphers = ha_proxy_ssl_ciphers - self.ha_proxy_ssl_options = ha_proxy_ssl_options + self.virtual_ip = virtual_ip self.haproxy_container_image = haproxy_container_image self.keepalived_container_image = keepalived_container_image - # placeholder variable. Need definitive list of hosts this service will - # be placed on in order to generate keepalived config. Will be populated - # when applying spec - self.definitive_host_list = [] # type: List[str] + + def get_port_start(self) -> List[int]: + return [cast(int, self.frontend_port), + cast(int, self.monitor_port)] def validate(self) -> None: - super(HA_RGWSpec, self).validate() + super(IngressSpec, self).validate() - if not self.virtual_ip_interface: - raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No Virtual IP Interface specified') - if not self.virtual_ip_address: + if not self.backend_service: raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No Virtual IP Address specified') - if not self.frontend_port and not self.ha_proxy_frontend_ssl_certificate: + 'Cannot add ingress: No backend_service specified') + if not self.frontend_port: raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No Frontend Port specified') - if not self.ha_proxy_port: + 'Cannot add ingress: No frontend_port specified') + if not self.monitor_port: raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No HA Proxy Port specified') - if not self.ha_proxy_stats_enabled: + 'Cannot add ingress: No monitor_port specified') + if not self.virtual_ip: raise ServiceSpecValidationError( - 'Cannot add ha-rgw: Ha Proxy Stats Enabled option not set') - if not self.ha_proxy_stats_user: - raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No HA Proxy Stats User specified') - if not self.ha_proxy_stats_password: - raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No HA Proxy Stats Password specified') - if not self.ha_proxy_enable_prometheus_exporter: - raise ServiceSpecValidationError( - 'Cannot add ha-rgw: HA Proxy Enable Prometheus Exporter option not set') - if not self.ha_proxy_monitor_uri: - raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No HA Proxy Monitor Uri specified') - if not self.keepalived_password: - raise ServiceSpecValidationError( - 'Cannot add ha-rgw: No Keepalived Password specified') - if self.ha_proxy_frontend_ssl_certificate: - if not self.ha_proxy_frontend_ssl_port: - raise ServiceSpecValidationError( - 'Cannot add ha-rgw: Specified Ha Proxy Frontend SSL ' + - 'Certificate but no SSL Port') + 'Cannot add ingress: No virtual_ip provided') class CustomContainerSpec(ServiceSpec): -- 2.39.5