From 7094d79f5e38ce5cc0b333f11a19715abbadfcdd Mon Sep 17 00:00:00 2001 From: Luis Domingues Date: Tue, 13 Jun 2023 09:59:35 +0200 Subject: [PATCH] cephadm: Split multicast interface and unicast_ip in keepalived.conf When deploying keepalived, cephadm set interface and unicast_src_ip in keepalived.conf. However, having both options are set, but if unicast_src_ip is not in the interface set by 'interface', the instrances of keepalived will not properly commuicate. This commit makes both options exclusive, and add an option to set either one or the other. Default is set to 'interface', as it seems multicast is the default way to deploy keepalived. Signed-off-by: Luis Domingues (cherry picked from commit 0dfe3e68a9284736f643a06acfda719cec0a6ea4) (cherry picked from commit 837c273da2aa3dd3b3fe09b9ac04fd032149de6d) --- doc/cephadm/services/rgw.rst | 24 ++++++++++++++----- src/pybind/mgr/cephadm/services/ingress.py | 19 +++++++++++++++ .../services/ingress/keepalived.conf.j2 | 4 +++- .../ceph/deployment/service_spec.py | 4 ++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst index 818648cf5fe..740f74bb9b5 100644 --- a/doc/cephadm/services/rgw.rst +++ b/doc/cephadm/services/rgw.rst @@ -239,12 +239,14 @@ It is a yaml format file with the following properties: - host2 - host3 spec: - backend_service: rgw.something # adjust to match your existing RGW service - virtual_ip: / # ex: 192.168.20.1/24 - frontend_port: # ex: 8080 - monitor_port: # ex: 1967, used by haproxy for load balancer status - virtual_interface_networks: [ ... ] # optional: list of CIDR networks - ssl_cert: | # optional: SSL certificate and key + backend_service: rgw.something # adjust to match your existing RGW service + virtual_ip: / # ex: 192.168.20.1/24 + frontend_port: # ex: 8080 + monitor_port: # ex: 1967, used by haproxy for load balancer status + virtual_interface_networks: [ ... ] # optional: list of CIDR networks + use_keepalived_multicast: # optional: Default is False. + vrrp_interface_network: / # optional: ex: 192.168.20.0/24 + ssl_cert: | # optional: SSL certificate and key -----BEGIN CERTIFICATE----- ... -----END CERTIFICATE----- @@ -303,6 +305,16 @@ where the properties of this service specification are: * ``ssl_cert``: SSL certificate, if SSL is to be enabled. This must contain the both the certificate and private key blocks in .pem format. +* ``use_keepalived_multicast`` + Default is False. By default, cephadm will deploy keepalived config to use unicast IPs, + using the IPs of the hosts. The IPs chosen will be the same IPs cephadm uses to connect + to the machines. But if multicast is prefered, we can set ``use_keepalived_multicast`` + to ``True`` and Keepalived will use multicast IP (224.0.0.18) to communicate between instances, + using the same interfaces as where the VIPs are. +* ``vrrp_interface_network`` + By default, cephadm will configure keepalived to use the same interface where the VIPs are + for VRRP communication. If another interface is needed, it can be set via ``vrrp_interface_network`` + with a network to identify which ethernet interface to use. .. _ingress-virtual-ip: diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index 720a0bf305c..be68180ac83 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -284,6 +284,24 @@ class IngressService(CephService): f"Unable to identify interface for {spec.virtual_ip} on {host}" ) + # Use interface as vrrp_interface for vrrp traffic if vrrp_interface_network not set on the spec + vrrp_interface = None + if not spec.vrrp_interface_network: + vrrp_interface = interface + else: + for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): + if subnet == spec.vrrp_interface_network: + vrrp_interface = list(ifaces.keys())[0] + logger.info( + f'vrrp will be configured on {host} interface ' + f'{vrrp_interface} (which has guiding subnet {subnet})' + ) + break + else: + raise OrchestratorError( + f"Unable to identify vrrp interface for {spec.vrrp_interface_network} on {host}" + ) + # script to monitor health script = '/usr/bin/false' for d in daemons: @@ -336,6 +354,7 @@ class IngressService(CephService): 'script': script, 'password': password, 'interface': interface, + 'vrrp_interface': vrrp_interface, 'virtual_ips': virtual_ips, 'states': states, 'priorities': priorities, diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 index f560c975665..006db52ea11 100644 --- a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 +++ b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 @@ -11,19 +11,21 @@ vrrp_script check_backend { vrrp_instance VI_{{ x }} { state {{ states[x] }} priority {{ priorities[x] }} - interface {{ interface }} + interface {{ vrrp_interface }} virtual_router_id {{ 50 + x }} advert_int 1 authentication { auth_type PASS auth_pass {{ password }} } +{% if not spec.use_keepalived_multicast %} unicast_src_ip {{ host_ip }} unicast_peer { {% for ip in other_ips %} {{ ip }} {% endfor %} } +{% endif %} virtual_ipaddress { {{ virtual_ips[x] }} dev {{ interface }} } diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index e950c48a06d..ea8a012bba3 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -1284,6 +1284,8 @@ class IngressSpec(ServiceSpec): virtual_ip: Optional[str] = None, virtual_ips_list: Optional[List[str]] = None, virtual_interface_networks: Optional[List[str]] = [], + use_keepalived_multicast: Optional[bool] = False, + vrrp_interface_network: Optional[str] = None, unmanaged: bool = False, ssl: bool = False, keepalive_only: bool = False, @@ -1316,6 +1318,8 @@ class IngressSpec(ServiceSpec): self.virtual_ip = virtual_ip self.virtual_ips_list = virtual_ips_list self.virtual_interface_networks = virtual_interface_networks or [] + self.use_keepalived_multicast = use_keepalived_multicast + self.vrrp_interface_network = vrrp_interface_network self.unmanaged = unmanaged self.ssl = ssl self.keepalive_only = keepalive_only -- 2.39.5