cephadm: Split multicast interface and unicast_ip in keepalived.conf

When deploying keepalived, cephadm set interface and unicast_src_ip
in keepalived.conf. However, having both options are set, but
if unicast_src_ip is not in the interface set by 'interface',
the instrances of keepalived will not properly commuicate.

This commit makes both options exclusive, and add an option to set
either one or the other. Default is set to 'interface', as it seems
multicast is the default way to deploy keepalived.

Signed-off-by: Luis Domingues <domingues.luis@protonmail.ch>

diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst
index 818648cf5fee46bd7145f0d42c813eec5b6a7383..740f74bb9b5d808f3e51b1f36ef849d32169b6bc 100644 (file)
--- a/doc/cephadm/services/rgw.rst
+++ b/doc/cephadm/services/rgw.rst
@@ -239,12 +239,14 @@ It is a yaml format file with the following properties:
         - host2
         - host3
     spec:
-      backend_service: rgw.something      # adjust to match your existing RGW service
-      virtual_ip: <string>/<string>       # ex: 192.168.20.1/24
-      frontend_port: <integer>            # ex: 8080
-      monitor_port: <integer>             # ex: 1967, used by haproxy for load balancer status
-      virtual_interface_networks: [ ... ] # optional: list of CIDR networks
-      ssl_cert: |                         # optional: SSL certificate and key
+      backend_service: rgw.something            # adjust to match your existing RGW service
+      virtual_ip: <string>/<string>             # ex: 192.168.20.1/24
+      frontend_port: <integer>                  # ex: 8080
+      monitor_port: <integer>                   # ex: 1967, used by haproxy for load balancer status
+      virtual_interface_networks: [ ... ]       # optional: list of CIDR networks
+      use_keepalived_multicast: <bool>          # optional: Default is False.
+      vrrp_interface_network: <string>/<string> # optional: ex: 192.168.20.0/24
+      ssl_cert: |                               # optional: SSL certificate and key
         -----BEGIN CERTIFICATE-----
         ...
         -----END CERTIFICATE-----
@@ -303,6 +305,16 @@ where the properties of this service specification are:
 * ``ssl_cert``:
     SSL certificate, if SSL is to be enabled. This must contain the both the certificate and
     private key blocks in .pem format.
+* ``use_keepalived_multicast``
+    Default is False. By default, cephadm will deploy keepalived config to use unicast IPs,
+    using the IPs of the hosts. The IPs chosen will be the same IPs cephadm uses to connect
+    to the machines. But if multicast is prefered, we can set ``use_keepalived_multicast``
+    to ``True`` and Keepalived will use multicast IP (224.0.0.18) to communicate between instances,
+    using the same interfaces as where the VIPs are.
+* ``vrrp_interface_network``
+    By default, cephadm will configure keepalived to use the same interface where the VIPs are
+    for VRRP communication. If another interface is needed, it can be set via ``vrrp_interface_network``
+    with a network to identify which ethernet interface to use.
 
 .. _ingress-virtual-ip:
 

diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py
index 95cff4bad194460c234f9a0be18f73982c133921..664c291c46abffacffef982f7da84725fd2048cc 100644 (file)
--- a/src/pybind/mgr/cephadm/services/ingress.py
+++ b/src/pybind/mgr/cephadm/services/ingress.py
@@ -280,6 +280,24 @@ class IngressService(CephService):
                 f"Unable to identify interface for {spec.virtual_ip} on {host}"
             )
 
+        # Use interface as vrrp_interface for vrrp traffic if vrrp_interface_network not set on the spec
+        vrrp_interface = None
+        if not spec.vrrp_interface_network:
+            vrrp_interface = interface
+        else:
+            for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items():
+                if subnet == spec.vrrp_interface_network:
+                    vrrp_interface = list(ifaces.keys())[0]
+                    logger.info(
+                        f'vrrp will be configured on {host} interface '
+                        f'{vrrp_interface} (which has guiding subnet {subnet})'
+                    )
+                    break
+            else:
+                raise OrchestratorError(
+                    f"Unable to identify vrrp interface for {spec.vrrp_interface_network} on {host}"
+                )
+
         # script to monitor health
         script = '/usr/bin/false'
         for d in daemons:
@@ -332,6 +350,7 @@ class IngressService(CephService):
                 'script': script,
                 'password': password,
                 'interface': interface,
+                'vrrp_interface': vrrp_interface,
                 'virtual_ips': virtual_ips,
                 'states': states,
                 'priorities': priorities,

diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2
index f560c9756654d0d51a025dc37d668ee7575023da..006db52ea112ac776017a0dc93fdaf07305d393b 100644 (file)
--- a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2
@@ -11,19 +11,21 @@ vrrp_script check_backend {
 vrrp_instance VI_{{ x }} {
   state {{ states[x] }}
   priority {{ priorities[x] }}
-  interface {{ interface }}
+  interface {{ vrrp_interface }}
   virtual_router_id {{ 50 + x }}
   advert_int 1
   authentication {
       auth_type PASS
       auth_pass {{ password }}
   }
+{% if not spec.use_keepalived_multicast %}
   unicast_src_ip {{ host_ip }}
   unicast_peer {
     {% for ip in other_ips %}
     {{ ip }}
     {% endfor %}
   }
+{% endif %}
   virtual_ipaddress {
     {{ virtual_ips[x] }} dev {{ interface }}
   }

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 4937318261a5aff1c086caefb40d1a13c38fd48c..232586bead3f14abb83c68afb7488c61dcdd3656 100644 (file)
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1053,6 +1053,8 @@ class IngressSpec(ServiceSpec):
                  virtual_ip: Optional[str] = None,
                  virtual_ips_list: Optional[List[str]] = None,
                  virtual_interface_networks: Optional[List[str]] = [],
+                 use_keepalived_multicast: Optional[bool] = False,
+                 vrrp_interface_network: Optional[str] = None,
                  unmanaged: bool = False,
                  ssl: bool = False,
                  keepalive_only: bool = False,
@@ -1085,6 +1087,8 @@ class IngressSpec(ServiceSpec):
         self.virtual_ip = virtual_ip
         self.virtual_ips_list = virtual_ips_list
         self.virtual_interface_networks = virtual_interface_networks or []
+        self.use_keepalived_multicast = use_keepalived_multicast
+        self.vrrp_interface_network = vrrp_interface_network
         self.unmanaged = unmanaged
         self.ssl = ssl
         self.keepalive_only = keepalive_only