mgr/cephadm: add tcp mode for ingress over RGW

In order to allow TLS for the full path client -> haproxy
-> RGW, give the option for the ingress service to just
serve in tcp mode and pass received data directly
to the backend servers. Makes no difference for the
ingress over nfs case that already used tcp mode.

Signed-off-by: Adam King <adking@redhat.com>

diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py
index 48c1a3949183279bb18db5940c648bd475af9d9b..01febfe7a708d280d2594a0b71a79950b4c1ddd6 100644 (file)
--- a/src/pybind/mgr/cephadm/services/ingress.py
+++ b/src/pybind/mgr/cephadm/services/ingress.py
@@ -203,7 +203,7 @@ class IngressService(CephService):
                         'port': 0,
                     })
         else:
-            mode = 'http'
+            mode = 'tcp' if spec.use_tcp_mode_over_rgw else 'http'
             servers = [
                 {
                     'name': d.name(),

diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
index b9b528437689cab90f4d20f41aa0bd0a5bbc2c3b..6ced000389789933a3055d30222f0ada12a26f5d 100644 (file)
--- a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
+++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
@@ -68,6 +68,9 @@ frontend frontend
     bind {{ ip }}:{{ frontend_port }} ssl crt /var/lib/haproxy/haproxy.pem {{ v4v6_flag }}
 {% else %}
     bind {{ ip }}:{{ frontend_port }} {{ v4v6_flag }}
+{% endif %}
+{% if mode == 'tcp' %}
+    option tcplog
 {% endif %}
     default_backend backend
 
@@ -80,16 +83,24 @@ backend backend
 {% endif %}
     balance static-rr
     option httpchk HEAD / HTTP/1.0
-    {% for server in servers %}
-    server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 inter {{ health_check_interval }}
-    {% endfor %}
 {% endif %}
 {% if mode == 'tcp' %}
     mode        tcp
     balance     roundrobin
+{% if spec.use_tcp_mode_over_rgw %}
+{% if backend_spec.ssl %}
+    option ssl-hello-chk
+{% endif %}
+{% endif %}
 {% if default_server_opts %}
     default-server {{ default_server_opts|join(" ") }}
 {% endif %}
+{% endif %}
+{% if backend_spec.service_type == 'rgw' %}
+    {% for server in servers %}
+    server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 inter {{ health_check_interval }}
+    {% endfor %}
+{% else %}
     {% for server in servers %}
     server {{ server.name }} {{ server.ip }}:{{ server.port }} check
     {% endfor %}

diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index cd1e98b0df2ff0a118605e923057beb77d441302..436dee7909fedd1ce2d11a599cbbc7375f11d551 100644 (file)
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -2783,6 +2783,7 @@ class TestIngressService:
             '    monitor-uri /health\n\n'
             'frontend frontend\n'
             '    bind 192.168.122.100:2049\n'
+            '    option tcplog\n'
             '    default_backend backend\n\n'
             'backend backend\n'
             '    mode        tcp\n'
@@ -3095,6 +3096,82 @@ class TestIngressService:
                 exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()]
                 assert gen_config_lines == exp_config_lines
 
+    @patch("cephadm.serve.CephadmServe._run_cephadm")
+    def test_haproxy_config_rgw_tcp_mode(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
+        _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+        with with_host(cephadm_module, 'test'):
+            cephadm_module.cache.update_host_networks('test', {
+                '1.2.3.0/24': {
+                    'if0': ['1.2.3.1']
+                }
+            })
+
+            # the ingress backend
+            s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1),
+                        rgw_frontend_type='beast', rgw_frontend_port=443, ssl=True)
+
+            ispec = IngressSpec(service_type='ingress',
+                                service_id='test',
+                                backend_service='rgw.foo',
+                                frontend_port=8089,
+                                monitor_port=8999,
+                                monitor_user='admin',
+                                monitor_password='12345',
+                                virtual_interface_networks=['1.2.3.0/24'],
+                                virtual_ip="1.2.3.4/32",
+                                use_tcp_mode_over_rgw=True)
+            with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _:
+                # generate the haproxy conf based on the specified spec
+                haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config(
+                    CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name()))
+
+                haproxy_expected_conf = {
+                    'files':
+                        {
+                            'haproxy.cfg':
+                                '# This file is generated by cephadm.'
+                                '\nglobal\n    log         '
+                                '127.0.0.1 local2\n    '
+                                'chroot      /var/lib/haproxy\n    '
+                                'pidfile     /var/lib/haproxy/haproxy.pid\n    '
+                                'maxconn     8000\n    '
+                                'daemon\n    '
+                                'stats socket /var/lib/haproxy/stats\n'
+                                '\ndefaults\n    '
+                                'mode                    tcp\n    '
+                                'log                     global\n    '
+                                'timeout queue           1m\n    '
+                                'timeout connect         10s\n    '
+                                'timeout client          1m\n    '
+                                'timeout server          1m\n    '
+                                'timeout check           10s\n    '
+                                'maxconn                 8000\n'
+                                '\nfrontend stats\n    '
+                                'mode http\n    '
+                                'bind 1.2.3.4:8999\n    '
+                                'bind 1::4:8999\n    '
+                                'stats enable\n    '
+                                'stats uri /stats\n    '
+                                'stats refresh 10s\n    '
+                                'stats auth admin:12345\n    '
+                                'http-request use-service prometheus-exporter if { path /metrics }\n    '
+                                'monitor-uri /health\n'
+                                '\nfrontend frontend\n    '
+                                'bind 1.2.3.4:8089 \n    '
+                                'option tcplog\n    '
+                                'default_backend backend\n\n'
+                                'backend backend\n    '
+                                'mode        tcp\n    '
+                                'balance     roundrobin\n    '
+                                'hash-type   consistent\n    '
+                                'option ssl-hello-chk\n    '
+                                'server '
+                                + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100 inter 2s\n'
+                        }
+                }
+
+                assert haproxy_generated_conf[0] == haproxy_expected_conf
+
     @patch("cephadm.serve.CephadmServe._run_cephadm")
     def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
         _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
@@ -3602,6 +3679,7 @@ class TestIngressService:
             '    monitor-uri /health\n\n'
             'frontend frontend\n'
             '    bind 192.168.122.100:2049\n'
+            '    option tcplog\n'
             '    default_backend backend\n\n'
             'backend backend\n'
             '    mode        tcp\n'

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index ff10b5fd2dee725e61bc41194b1f8a0c173a3b5a..853e3af9571fff1c6910f25f968649479a9bdfc7 100644 (file)
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -2213,6 +2213,7 @@ class IngressSpec(ServiceSpec):
                  monitor_cert_source: Optional[str] = MonitorCertSource.REUSE_SERVICE_CERT.value,
                  monitor_networks: Optional[List[str]] = None,
                  monitor_ip_addrs: Optional[Dict[str, str]] = None,
+                 use_tcp_mode_over_rgw: bool = False,
                  ):
         assert service_type == 'ingress'
 
@@ -2257,6 +2258,7 @@ class IngressSpec(ServiceSpec):
         self.monitor_cert_source = monitor_cert_source
         self.monitor_networks = monitor_networks
         self.monitor_ip_addrs = monitor_ip_addrs
+        self.use_tcp_mode_over_rgw = use_tcp_mode_over_rgw
 
     def get_port_start(self) -> List[int]:
         ports = []