From: Luis Domingues Date: Tue, 13 Jun 2023 07:59:35 +0000 (+0200) Subject: cephadm: Split multicast interface and unicast_ip in keepalived.conf X-Git-Tag: v19.0.0~906^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0dfe3e68a9284736f643a06acfda719cec0a6ea4;p=ceph.git cephadm: Split multicast interface and unicast_ip in keepalived.conf When deploying keepalived, cephadm set interface and unicast_src_ip in keepalived.conf. However, having both options are set, but if unicast_src_ip is not in the interface set by 'interface', the instrances of keepalived will not properly commuicate. This commit makes both options exclusive, and add an option to set either one or the other. Default is set to 'interface', as it seems multicast is the default way to deploy keepalived. Signed-off-by: Luis Domingues --- diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst index 818648cf5fe..740f74bb9b5 100644 --- a/doc/cephadm/services/rgw.rst +++ b/doc/cephadm/services/rgw.rst @@ -239,12 +239,14 @@ It is a yaml format file with the following properties: - host2 - host3 spec: - backend_service: rgw.something # adjust to match your existing RGW service - virtual_ip: / # ex: 192.168.20.1/24 - frontend_port: # ex: 8080 - monitor_port: # ex: 1967, used by haproxy for load balancer status - virtual_interface_networks: [ ... ] # optional: list of CIDR networks - ssl_cert: | # optional: SSL certificate and key + backend_service: rgw.something # adjust to match your existing RGW service + virtual_ip: / # ex: 192.168.20.1/24 + frontend_port: # ex: 8080 + monitor_port: # ex: 1967, used by haproxy for load balancer status + virtual_interface_networks: [ ... ] # optional: list of CIDR networks + use_keepalived_multicast: # optional: Default is False. + vrrp_interface_network: / # optional: ex: 192.168.20.0/24 + ssl_cert: | # optional: SSL certificate and key -----BEGIN CERTIFICATE----- ... -----END CERTIFICATE----- @@ -303,6 +305,16 @@ where the properties of this service specification are: * ``ssl_cert``: SSL certificate, if SSL is to be enabled. This must contain the both the certificate and private key blocks in .pem format. +* ``use_keepalived_multicast`` + Default is False. By default, cephadm will deploy keepalived config to use unicast IPs, + using the IPs of the hosts. The IPs chosen will be the same IPs cephadm uses to connect + to the machines. But if multicast is prefered, we can set ``use_keepalived_multicast`` + to ``True`` and Keepalived will use multicast IP (224.0.0.18) to communicate between instances, + using the same interfaces as where the VIPs are. +* ``vrrp_interface_network`` + By default, cephadm will configure keepalived to use the same interface where the VIPs are + for VRRP communication. If another interface is needed, it can be set via ``vrrp_interface_network`` + with a network to identify which ethernet interface to use. .. _ingress-virtual-ip: diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index 95cff4bad19..664c291c46a 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -280,6 +280,24 @@ class IngressService(CephService): f"Unable to identify interface for {spec.virtual_ip} on {host}" ) + # Use interface as vrrp_interface for vrrp traffic if vrrp_interface_network not set on the spec + vrrp_interface = None + if not spec.vrrp_interface_network: + vrrp_interface = interface + else: + for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): + if subnet == spec.vrrp_interface_network: + vrrp_interface = list(ifaces.keys())[0] + logger.info( + f'vrrp will be configured on {host} interface ' + f'{vrrp_interface} (which has guiding subnet {subnet})' + ) + break + else: + raise OrchestratorError( + f"Unable to identify vrrp interface for {spec.vrrp_interface_network} on {host}" + ) + # script to monitor health script = '/usr/bin/false' for d in daemons: @@ -332,6 +350,7 @@ class IngressService(CephService): 'script': script, 'password': password, 'interface': interface, + 'vrrp_interface': vrrp_interface, 'virtual_ips': virtual_ips, 'states': states, 'priorities': priorities, diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 index f560c975665..006db52ea11 100644 --- a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 +++ b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 @@ -11,19 +11,21 @@ vrrp_script check_backend { vrrp_instance VI_{{ x }} { state {{ states[x] }} priority {{ priorities[x] }} - interface {{ interface }} + interface {{ vrrp_interface }} virtual_router_id {{ 50 + x }} advert_int 1 authentication { auth_type PASS auth_pass {{ password }} } +{% if not spec.use_keepalived_multicast %} unicast_src_ip {{ host_ip }} unicast_peer { {% for ip in other_ips %} {{ ip }} {% endfor %} } +{% endif %} virtual_ipaddress { {{ virtual_ips[x] }} dev {{ interface }} } diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 4937318261a..232586bead3 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -1053,6 +1053,8 @@ class IngressSpec(ServiceSpec): virtual_ip: Optional[str] = None, virtual_ips_list: Optional[List[str]] = None, virtual_interface_networks: Optional[List[str]] = [], + use_keepalived_multicast: Optional[bool] = False, + vrrp_interface_network: Optional[str] = None, unmanaged: bool = False, ssl: bool = False, keepalive_only: bool = False, @@ -1085,6 +1087,8 @@ class IngressSpec(ServiceSpec): self.virtual_ip = virtual_ip self.virtual_ips_list = virtual_ips_list self.virtual_interface_networks = virtual_interface_networks or [] + self.use_keepalived_multicast = use_keepalived_multicast + self.vrrp_interface_network = vrrp_interface_network self.unmanaged = unmanaged self.ssl = ssl self.keepalive_only = keepalive_only