From 1753b43c710c799793a1d72b44adc771e717c842 Mon Sep 17 00:00:00 2001 From: Frank Ederveen Date: Mon, 14 Nov 2022 12:27:45 +0000 Subject: [PATCH] mgr/cephadm: add ingress support for ssl rgw service This commit adds support for ssl backend rgw. This allows for full network encryption. Fixes: https://tracker.ceph.com/issues/51972 Signed-off-by: Frank Ederveen (cherry picked from commit ab9d6ea42453225189f3ded8863c00bc55b71189) --- doc/cephadm/services/rgw.rst | 9 +- src/pybind/mgr/cephadm/services/ingress.py | 1 + .../templates/services/ingress/haproxy.cfg.j2 | 4 + src/pybind/mgr/cephadm/tests/test_services.py | 125 ++++++++++++++++++ 4 files changed, 135 insertions(+), 4 deletions(-) diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst index 58318b727e145..d0a58000130c0 100644 --- a/doc/cephadm/services/rgw.rst +++ b/doc/cephadm/services/rgw.rst @@ -164,8 +164,10 @@ for RGW with a minimum set of configuration options. The orchestrator will deploy and manage a combination of haproxy and keepalived to provide load balancing on a floating virtual IP. -If SSL is used, then SSL must be configured and terminated by the ingress service -and not RGW itself. +If the RGW service is configured with SSL enabled, then the ingress service +will use the `ssl` and `verify none` options in the backend configuration. +Trust verification is disabled because the backends are accessed by IP +address instead of FQDN. .. image:: ../../images/HAProxy_for_RGW.svg @@ -186,8 +188,7 @@ between all the RGW daemons available. Prerequisites ------------- -* An existing RGW service, without SSL. (If you want SSL service, the certificate - should be configured on the ingress service, not the RGW service.) +* An existing RGW service. Deploying --------- diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index 75276535ac717..d53d719c287d3 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -127,6 +127,7 @@ class IngressService(CephService): 'services/ingress/haproxy.cfg.j2', { 'spec': spec, + 'backend_spec': backend_spec, 'mode': mode, 'servers': servers, 'user': spec.monitor_user or 'admin', diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 index 01db8a50a3c8f..6d64c63634587 100644 --- a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 +++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 @@ -67,6 +67,10 @@ frontend frontend backend backend {% if mode == 'http' %} option forwardfor +{% if backend_spec.ssl %} + default-server ssl + default-server verify none +{% endif %} balance static-rr option httpchk HEAD / HTTP/1.0 {% for server in servers %} diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py index f834893abf2ce..3b7cb345413a2 100644 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ b/src/pybind/mgr/cephadm/tests/test_services.py @@ -1068,6 +1068,131 @@ class TestIngressService: assert haproxy_generated_conf[0] == haproxy_expected_conf + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_ingress_config_ssl_rgw(self, _run_cephadm, cephadm_module: CephadmOrchestrator): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + + with with_host(cephadm_module, 'test'): + cephadm_module.cache.update_host_networks('test', { + '1.2.3.0/24': { + 'if0': ['1.2.3.4/32'] + } + }) + + # the ingress backend + s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), + rgw_frontend_type='beast', rgw_frontend_port=443, ssl=True) + + ispec = IngressSpec(service_type='ingress', + service_id='test', + backend_service='rgw.foo', + frontend_port=8089, + monitor_port=8999, + monitor_user='admin', + monitor_password='12345', + keepalived_password='12345', + virtual_interface_networks=['1.2.3.0/24'], + virtual_ip="1.2.3.4/32") + with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: + # generate the keepalived conf based on the specified spec + keepalived_generated_conf = cephadm_module.cephadm_services['ingress'].keepalived_generate_config( + CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) + + keepalived_expected_conf = { + 'files': + { + 'keepalived.conf': + '# This file is generated by cephadm.\n' + 'vrrp_script check_backend {\n ' + 'script "/usr/bin/curl http://localhost:8999/health"\n ' + 'weight -20\n ' + 'interval 2\n ' + 'rise 2\n ' + 'fall 2\n}\n\n' + 'vrrp_instance VI_0 {\n ' + 'state MASTER\n ' + 'priority 100\n ' + 'interface if0\n ' + 'virtual_router_id 50\n ' + 'advert_int 1\n ' + 'authentication {\n ' + 'auth_type PASS\n ' + 'auth_pass 12345\n ' + '}\n ' + 'unicast_src_ip 1::4\n ' + 'unicast_peer {\n ' + '}\n ' + 'virtual_ipaddress {\n ' + '1.2.3.4/32 dev if0\n ' + '}\n ' + 'track_script {\n ' + 'check_backend\n }\n' + '}\n' + } + } + + # check keepalived config + assert keepalived_generated_conf[0] == keepalived_expected_conf + + # generate the haproxy conf based on the specified spec + haproxy_generated_conf = cephadm_module.cephadm_services['ingress'].haproxy_generate_config( + CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) + + haproxy_expected_conf = { + 'files': + { + 'haproxy.cfg': + '# This file is generated by cephadm.' + '\nglobal\n log ' + '127.0.0.1 local2\n ' + 'chroot /var/lib/haproxy\n ' + 'pidfile /var/lib/haproxy/haproxy.pid\n ' + 'maxconn 8000\n ' + 'daemon\n ' + 'stats socket /var/lib/haproxy/stats\n' + '\ndefaults\n ' + 'mode http\n ' + 'log global\n ' + 'option httplog\n ' + 'option dontlognull\n ' + 'option http-server-close\n ' + 'option forwardfor except 127.0.0.0/8\n ' + 'option redispatch\n ' + 'retries 3\n ' + 'timeout queue 20s\n ' + 'timeout connect 5s\n ' + 'timeout http-request 1s\n ' + 'timeout http-keep-alive 5s\n ' + 'timeout client 30s\n ' + 'timeout server 30s\n ' + 'timeout check 5s\n ' + 'maxconn 8000\n' + '\nfrontend stats\n ' + 'mode http\n ' + 'bind 1.2.3.4:8999\n ' + 'bind localhost:8999\n ' + 'stats enable\n ' + 'stats uri /stats\n ' + 'stats refresh 10s\n ' + 'stats auth admin:12345\n ' + 'http-request use-service prometheus-exporter if { path /metrics }\n ' + 'monitor-uri /health\n' + '\nfrontend frontend\n ' + 'bind 1.2.3.4:8089\n ' + 'default_backend backend\n\n' + 'backend backend\n ' + 'option forwardfor\n ' + 'default-server ssl\n ' + 'default-server verify none\n ' + 'balance static-rr\n ' + 'option httpchk HEAD / HTTP/1.0\n ' + 'server ' + + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100\n' + } + } + + assert haproxy_generated_conf[0] == haproxy_expected_conf + @patch("cephadm.serve.CephadmServe._run_cephadm") def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOrchestrator): _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) -- 2.39.5