cephadm: Split multicast interface and unicast_ip in keepalived.conf

When deploying keepalived, cephadm set interface and unicast_src_ip
in keepalived.conf. However, having both options are set, but
if unicast_src_ip is not in the interface set by 'interface',
the instrances of keepalived will not properly commuicate.

This commit makes both options exclusive, and add an option to set
either one or the other. Default is set to 'interface', as it seems
multicast is the default way to deploy keepalived.

Signed-off-by: Luis Domingues <domingues.luis@protonmail.ch>
(cherry picked from commit 0dfe3e68a9284736f643a06acfda719cec0a6ea4)
(cherry picked from commit 837c273da2aa3dd3b3fe09b9ac04fd032149de6d)

diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst
index 818648cf5fee46bd7145f0d42c813eec5b6a7383..740f74bb9b5d808f3e51b1f36ef849d32169b6bc 100644 (file)
--- a/doc/cephadm/services/rgw.rst
+++ b/doc/cephadm/services/rgw.rst
@@ -239,12 +239,14 @@ It is a yaml format file with the following properties:
         - host2
         - host3
     spec:
-      backend_service: rgw.something      # adjust to match your existing RGW service
-      virtual_ip: <string>/<string>       # ex: 192.168.20.1/24
-      frontend_port: <integer>            # ex: 8080
-      monitor_port: <integer>             # ex: 1967, used by haproxy for load balancer status
-      virtual_interface_networks: [ ... ] # optional: list of CIDR networks
-      ssl_cert: |                         # optional: SSL certificate and key
+      backend_service: rgw.something            # adjust to match your existing RGW service
+      virtual_ip: <string>/<string>             # ex: 192.168.20.1/24
+      frontend_port: <integer>                  # ex: 8080
+      monitor_port: <integer>                   # ex: 1967, used by haproxy for load balancer status
+      virtual_interface_networks: [ ... ]       # optional: list of CIDR networks
+      use_keepalived_multicast: <bool>          # optional: Default is False.
+      vrrp_interface_network: <string>/<string> # optional: ex: 192.168.20.0/24
+      ssl_cert: |                               # optional: SSL certificate and key
         -----BEGIN CERTIFICATE-----
         ...
         -----END CERTIFICATE-----
@@ -303,6 +305,16 @@ where the properties of this service specification are:
 * ``ssl_cert``:
     SSL certificate, if SSL is to be enabled. This must contain the both the certificate and
     private key blocks in .pem format.
+* ``use_keepalived_multicast``
+    Default is False. By default, cephadm will deploy keepalived config to use unicast IPs,
+    using the IPs of the hosts. The IPs chosen will be the same IPs cephadm uses to connect
+    to the machines. But if multicast is prefered, we can set ``use_keepalived_multicast``
+    to ``True`` and Keepalived will use multicast IP (224.0.0.18) to communicate between instances,
+    using the same interfaces as where the VIPs are.
+* ``vrrp_interface_network``
+    By default, cephadm will configure keepalived to use the same interface where the VIPs are
+    for VRRP communication. If another interface is needed, it can be set via ``vrrp_interface_network``
+    with a network to identify which ethernet interface to use.
 
 .. _ingress-virtual-ip:
 

diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py
index 720a0bf305c5726b89d1f5ed9409f342cc03179c..be68180ac83c9ce6369c2a3b307fc3f34b3bda88 100644 (file)
--- a/src/pybind/mgr/cephadm/services/ingress.py
+++ b/src/pybind/mgr/cephadm/services/ingress.py
@@ -284,6 +284,24 @@ class IngressService(CephService):
                 f"Unable to identify interface for {spec.virtual_ip} on {host}"
             )
 
+        # Use interface as vrrp_interface for vrrp traffic if vrrp_interface_network not set on the spec
+        vrrp_interface = None
+        if not spec.vrrp_interface_network:
+            vrrp_interface = interface
+        else:
+            for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items():
+                if subnet == spec.vrrp_interface_network:
+                    vrrp_interface = list(ifaces.keys())[0]
+                    logger.info(
+                        f'vrrp will be configured on {host} interface '
+                        f'{vrrp_interface} (which has guiding subnet {subnet})'
+                    )
+                    break
+            else:
+                raise OrchestratorError(
+                    f"Unable to identify vrrp interface for {spec.vrrp_interface_network} on {host}"
+                )
+
         # script to monitor health
         script = '/usr/bin/false'
         for d in daemons:
@@ -336,6 +354,7 @@ class IngressService(CephService):
                 'script': script,
                 'password': password,
                 'interface': interface,
+                'vrrp_interface': vrrp_interface,
                 'virtual_ips': virtual_ips,
                 'states': states,
                 'priorities': priorities,

diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2
index f560c9756654d0d51a025dc37d668ee7575023da..006db52ea112ac776017a0dc93fdaf07305d393b 100644 (file)
--- a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2
@@ -11,19 +11,21 @@ vrrp_script check_backend {
 vrrp_instance VI_{{ x }} {
   state {{ states[x] }}
   priority {{ priorities[x] }}
-  interface {{ interface }}
+  interface {{ vrrp_interface }}
   virtual_router_id {{ 50 + x }}
   advert_int 1
   authentication {
       auth_type PASS
       auth_pass {{ password }}
   }
+{% if not spec.use_keepalived_multicast %}
   unicast_src_ip {{ host_ip }}
   unicast_peer {
     {% for ip in other_ips %}
     {{ ip }}
     {% endfor %}
   }
+{% endif %}
   virtual_ipaddress {
     {{ virtual_ips[x] }} dev {{ interface }}
   }

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index e950c48a06dd7ff9edfe891e01eac8173304b343..ea8a012bba3778f4e68cd7eef347544df0594184 100644 (file)
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1284,6 +1284,8 @@ class IngressSpec(ServiceSpec):
                  virtual_ip: Optional[str] = None,
                  virtual_ips_list: Optional[List[str]] = None,
                  virtual_interface_networks: Optional[List[str]] = [],
+                 use_keepalived_multicast: Optional[bool] = False,
+                 vrrp_interface_network: Optional[str] = None,
                  unmanaged: bool = False,
                  ssl: bool = False,
                  keepalive_only: bool = False,
@@ -1316,6 +1318,8 @@ class IngressSpec(ServiceSpec):
         self.virtual_ip = virtual_ip
         self.virtual_ips_list = virtual_ips_list
         self.virtual_interface_networks = virtual_interface_networks or []
+        self.use_keepalived_multicast = use_keepalived_multicast
+        self.vrrp_interface_network = vrrp_interface_network
         self.unmanaged = unmanaged
         self.ssl = ssl
         self.keepalive_only = keepalive_only