From 7c67feb4db3dfdd6647aae048785838d8a94f225 Mon Sep 17 00:00:00 2001 From: Adam King Date: Mon, 15 Jul 2024 11:57:47 -0400 Subject: [PATCH] mgr/cephadm: add tcp mode for ingress over RGW In order to allow TLS for the full path client -> haproxy -> RGW, give the option for the ingress service to just serve in tcp mode and pass received data directly to the backend servers. Makes no difference for the ingress over nfs case that already used tcp mode. Signed-off-by: Adam King --- src/pybind/mgr/cephadm/services/ingress.py | 2 +- .../templates/services/ingress/haproxy.cfg.j2 | 17 +++- src/pybind/mgr/cephadm/tests/test_services.py | 78 +++++++++++++++++++ .../ceph/deployment/service_spec.py | 2 + 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index 48c1a39491832..01febfe7a708d 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -203,7 +203,7 @@ class IngressService(CephService): 'port': 0, }) else: - mode = 'http' + mode = 'tcp' if spec.use_tcp_mode_over_rgw else 'http' servers = [ { 'name': d.name(), diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 index b9b528437689c..6ced000389789 100644 --- a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 +++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 @@ -68,6 +68,9 @@ frontend frontend bind {{ ip }}:{{ frontend_port }} ssl crt /var/lib/haproxy/haproxy.pem {{ v4v6_flag }} {% else %} bind {{ ip }}:{{ frontend_port }} {{ v4v6_flag }} +{% endif %} +{% if mode == 'tcp' %} + option tcplog {% endif %} default_backend backend @@ -80,16 +83,24 @@ backend backend {% endif %} balance static-rr option httpchk HEAD / HTTP/1.0 - {% for server in servers %} - server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 inter {{ health_check_interval }} - {% endfor %} {% endif %} {% if mode == 'tcp' %} mode tcp balance roundrobin +{% if spec.use_tcp_mode_over_rgw %} +{% if backend_spec.ssl %} + option ssl-hello-chk +{% endif %} +{% endif %} {% if default_server_opts %} default-server {{ default_server_opts|join(" ") }} {% endif %} +{% endif %} +{% if backend_spec.service_type == 'rgw' %} + {% for server in servers %} + server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 inter {{ health_check_interval }} + {% endfor %} +{% else %} {% for server in servers %} server {{ server.name }} {{ server.ip }}:{{ server.port }} check {% endfor %} diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py index cd1e98b0df2ff..436dee7909fed 100644 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ b/src/pybind/mgr/cephadm/tests/test_services.py @@ -2783,6 +2783,7 @@ class TestIngressService: ' monitor-uri /health\n\n' 'frontend frontend\n' ' bind 192.168.122.100:2049\n' + ' option tcplog\n' ' default_backend backend\n\n' 'backend backend\n' ' mode tcp\n' @@ -3095,6 +3096,82 @@ class TestIngressService: exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] assert gen_config_lines == exp_config_lines + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_haproxy_config_rgw_tcp_mode(self, _run_cephadm, cephadm_module: CephadmOrchestrator): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + with with_host(cephadm_module, 'test'): + cephadm_module.cache.update_host_networks('test', { + '1.2.3.0/24': { + 'if0': ['1.2.3.1'] + } + }) + + # the ingress backend + s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), + rgw_frontend_type='beast', rgw_frontend_port=443, ssl=True) + + ispec = IngressSpec(service_type='ingress', + service_id='test', + backend_service='rgw.foo', + frontend_port=8089, + monitor_port=8999, + monitor_user='admin', + monitor_password='12345', + virtual_interface_networks=['1.2.3.0/24'], + virtual_ip="1.2.3.4/32", + use_tcp_mode_over_rgw=True) + with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: + # generate the haproxy conf based on the specified spec + haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( + CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) + + haproxy_expected_conf = { + 'files': + { + 'haproxy.cfg': + '# This file is generated by cephadm.' + '\nglobal\n log ' + '127.0.0.1 local2\n ' + 'chroot /var/lib/haproxy\n ' + 'pidfile /var/lib/haproxy/haproxy.pid\n ' + 'maxconn 8000\n ' + 'daemon\n ' + 'stats socket /var/lib/haproxy/stats\n' + '\ndefaults\n ' + 'mode tcp\n ' + 'log global\n ' + 'timeout queue 1m\n ' + 'timeout connect 10s\n ' + 'timeout client 1m\n ' + 'timeout server 1m\n ' + 'timeout check 10s\n ' + 'maxconn 8000\n' + '\nfrontend stats\n ' + 'mode http\n ' + 'bind 1.2.3.4:8999\n ' + 'bind 1::4:8999\n ' + 'stats enable\n ' + 'stats uri /stats\n ' + 'stats refresh 10s\n ' + 'stats auth admin:12345\n ' + 'http-request use-service prometheus-exporter if { path /metrics }\n ' + 'monitor-uri /health\n' + '\nfrontend frontend\n ' + 'bind 1.2.3.4:8089 \n ' + 'option tcplog\n ' + 'default_backend backend\n\n' + 'backend backend\n ' + 'mode tcp\n ' + 'balance roundrobin\n ' + 'hash-type consistent\n ' + 'option ssl-hello-chk\n ' + 'server ' + + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100 inter 2s\n' + } + } + + assert haproxy_generated_conf[0] == haproxy_expected_conf + @patch("cephadm.serve.CephadmServe._run_cephadm") def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOrchestrator): _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) @@ -3602,6 +3679,7 @@ class TestIngressService: ' monitor-uri /health\n\n' 'frontend frontend\n' ' bind 192.168.122.100:2049\n' + ' option tcplog\n' ' default_backend backend\n\n' 'backend backend\n' ' mode tcp\n' diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index ff10b5fd2dee7..853e3af9571ff 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -2213,6 +2213,7 @@ class IngressSpec(ServiceSpec): monitor_cert_source: Optional[str] = MonitorCertSource.REUSE_SERVICE_CERT.value, monitor_networks: Optional[List[str]] = None, monitor_ip_addrs: Optional[Dict[str, str]] = None, + use_tcp_mode_over_rgw: bool = False, ): assert service_type == 'ingress' @@ -2257,6 +2258,7 @@ class IngressSpec(ServiceSpec): self.monitor_cert_source = monitor_cert_source self.monitor_networks = monitor_networks self.monitor_ip_addrs = monitor_ip_addrs + self.use_tcp_mode_over_rgw = use_tcp_mode_over_rgw def get_port_start(self) -> List[int]: ports = [] -- 2.39.5