From b1aa0d52572b3d2372a1b6c8acd23a97bea51cf1 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Mon, 26 Apr 2021 15:29:45 -0400
Subject: [PATCH] mgr/cephadm: ingress: support nfs

- use consistent hashing
- statically map across ranks
- disable backend checks so that clients don't move

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit 51f0dedbbb996fdaca4ba866791c2ac86d4429c6)
---
 src/pybind/mgr/cephadm/services/ingress.py    | 51 ++++++++++++++++---
 .../templates/services/ingress/haproxy.cfg.j2 | 26 ++++++++--
 2 files changed, 66 insertions(+), 11 deletions(-)

diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py
index 639bef5a57db5..99e957bc7e321 100644
--- a/src/pybind/mgr/cephadm/services/ingress.py
+++ b/src/pybind/mgr/cephadm/services/ingress.py
@@ -64,6 +64,9 @@ class IngressService(CephService):
     ) -> Tuple[Dict[str, Any], List[str]]:
         spec = cast(IngressSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
         assert spec.backend_service
+        if spec.backend_service not in self.mgr.spec_store:
+            raise RuntimeError(f'{spec.service_name()} backend service {spec.backend_service} does not exist')
+        backend_spec = self.mgr.spec_store[spec.backend_service].spec
         daemons = self.mgr.cache.get_daemons_by_service(spec.backend_service)
         deps = [d.name() for d in daemons]
 
@@ -80,17 +83,48 @@ class IngressService(CephService):
         if spec.monitor_password:
             password = spec.monitor_password
 
+        if backend_spec.service_type == 'nfs':
+            mode = 'tcp'
+            by_rank = {d.rank: d for d in daemons if d.rank is not None}
+            servers = []
+
+            # try to establish how many ranks we *should* have
+            num_ranks = backend_spec.placement.count
+            if not num_ranks:
+                num_ranks = 1 + max(by_rank.keys())
+
+            for rank in range(num_ranks):
+                if rank in by_rank:
+                    d = by_rank[rank]
+                    assert(d.ports)
+                    servers.append({
+                        'name': f"{spec.backend_service}.{rank}",
+                        'ip': d.ip or resolve_ip(str(d.hostname)),
+                        'port': d.ports[0],
+                    })
+                else:
+                    # offline/missing server; leave rank in place
+                    servers.append({
+                        'name': f"{spec.backend_service}.{rank}",
+                        'ip': '0.0.0.0',
+                        'port': 0,
+                    })
+        else:
+            mode = 'http'
+            servers = [
+                {
+                    'name': d.name(),
+                    'ip': d.ip or resolve_ip(str(d.hostname)),
+                    'port': d.ports[0],
+                } for d in daemons if d.ports
+            ]
+
         haproxy_conf = self.mgr.template.render(
             'services/ingress/haproxy.cfg.j2',
             {
                 'spec': spec,
-                'servers': [
-                    {
-                        'name': d.name(),
-                        'ip': d.ip or resolve_ip(str(d.hostname)),
-                        'port': d.ports[0],
-                    } for d in daemons if d.ports
-                ],
+                'mode': mode,
+                'servers': servers,
                 'user': spec.monitor_user or 'admin',
                 'password': password,
                 'ip': daemon_spec.ip or '*',
@@ -194,7 +228,8 @@ class IngressService(CephService):
 
         # remove host, daemon is being deployed on from hosts list for
         # other_ips in conf file and converter to ips
-        hosts.remove(host)
+        if host in hosts:
+            hosts.remove(host)
         other_ips = [resolve_ip(h) for h in hosts]
 
         keepalived_conf = self.mgr.template.render(
diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
index 755067a2ca056..137dc1157edf9 100644
--- a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
+++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
@@ -19,24 +19,34 @@ global
 {% endif %}
 
 defaults
-    mode                    http
+    mode                    {{ mode }}
     log                     global
+{% if mode == 'http' %}
     option                  httplog
     option                  dontlognull
     option http-server-close
     option forwardfor       except 127.0.0.0/8
     option                  redispatch
     retries                 3
-    timeout http-request    1s
     timeout queue           20s
     timeout connect         5s
+    timeout http-request    1s
+    timeout http-keep-alive 5s
     timeout client          1s
     timeout server          1s
-    timeout http-keep-alive 5s
     timeout check           5s
+{% endif %}
+{% if mode == 'tcp' %}
+    timeout queue           1m
+    timeout connect         10s
+    timeout client          1m
+    timeout server          1m
+    timeout check           10s
+{% endif %}
     maxconn                 8000
 
 frontend stats
+    mode http
     bind {{ ip }}:{{ monitor_port }}
     stats enable
     stats uri /stats
@@ -54,9 +64,19 @@ frontend frontend
     default_backend backend
 
 backend backend
+{% if mode == 'http' %}
     option forwardfor
     balance static-rr
     option httpchk HEAD / HTTP/1.0
     {% for server in servers %}
     server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100
     {% endfor %}
+{% endif %}
+{% if mode == 'tcp' %}
+    mode        tcp
+    balance     source
+    hash-type   consistent
+    {% for server in servers %}
+    server {{ server.name }} {{ server.ip }}:{{ server.port }}
+    {% endfor %}
+{% endif %}
-- 
2.39.5