From: Shweta Bhosale Date: Wed, 3 Sep 2025 14:37:53 +0000 (+0530) Subject: mgr/cephadm: Allow Ingress service to expose the metrics via HTTPS also add fields... X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=da5d3ab0f32dc9e19da73ae01a0dbb95b2d1f8b6;p=ceph.git mgr/cephadm: Allow Ingress service to expose the metrics via HTTPS also add fields in spec to accept monitor ips/ monitor networks Fixes: https://tracker.ceph.com/issues/71707 Signed-off-by: Shweta Bhosale --- diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst index af647e9a8ebaa..33fde24619ba5 100644 --- a/doc/cephadm/services/rgw.rst +++ b/doc/cephadm/services/rgw.rst @@ -409,13 +409,30 @@ Service specs are YAML blocks with the following properties: use_keepalived_multicast: # optional: Default is False. vrrp_interface_network: / # optional: ex: 192.168.20.0/24 health_check_interval: # optional: Default is 2s. + ssl: true + certificate_source: inline # optional: Default is cephadm-signed ssl_cert: | # optional: SSL certificate and key -----BEGIN CERTIFICATE----- ... -----END CERTIFICATE----- + ssl_key: | -----BEGIN PRIVATE KEY----- ... -----END PRIVATE KEY----- + enable_stats: true + monitor_ssl: + monitor_cert_source: inline # optional: default is reuse_service_cert + monitor_ssl_cert: | # optional: SSL certificate and key + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- + monitor_ssl_key: | + -----BEGIN PRIVATE KEY----- + ... + -----END PRIVATE KEY----- + monitor_networks: [..] + monitor_ip_addrs: + host: .. code-block:: yaml @@ -467,9 +484,20 @@ where the properties of this service specification are: A list of networks to identify which ethernet interface to use for the virtual IP. * ``frontend_port`` The port used to access the ingress service. -* ``ssl_cert``: - SSL certificate, if SSL is to be enabled. This must contain the both the certificate and - private key blocks in .pem format. +* ``ssl`` + To enable SSL for ingress service. +* ``certificate_source`` + The certificate source can be one of the following: 'inline', 'reference', or 'cephadm-signed'. + - If set to 'inline', the YAML configuration must include ssl_cert and ssl_key. + - If set to 'reference', the certificate and key must already exist in the certificate store. + - If set to 'cephadm-signed', Cephadm will automatically generate the certificate and key. + By default, the source is set to 'cephadm-signed'. +* ``ssl_cert`` + SSL certificate, if SSL is enabled and ``certificate_source`` is not 'cephadm-signed'. + This should have the certificate .pem format. +* ``ssl_key`` + SSL key, if SSL is enabled and ``certificate_source`` is not 'cephadm-signed'. + This should have the key .pem format. * ``use_keepalived_multicast`` Default is False. By default, cephadm will deploy keepalived config to use unicast IPs, using the IPs of the hosts. The IPs chosen will be the same IPs cephadm uses to connect @@ -488,6 +516,29 @@ where the properties of this service specification are: * ``health_check_interval`` Default is 2 seconds. This parameter can be used to set the interval between health checks for the haproxy with the backend servers. +* ``enable_stats`` + Default is False, must be set to enable haproxy stats. +* ``monitor_ssl`` + To enable ssl for monitoring. SSL for monitoring can be enabled only when service SSL is enabled. +* ``monitor_cert_source`` + The monitor certificate source can be one of the following: 'reuse_service_cert', 'inline', 'reference', or 'cephadm-signed'. + - If set to 'reuse_service_cert', then the service certs will be used. + - If set to 'inline', the YAML configuration must include ssl_cert and ssl_key. + - If set to 'reference', the certificate and key must already exist in the certificate store. + - If set to 'cephadm-signed', Cephadm will automatically generate the certificate and key. + By default, the source is set to 'reuse_service_cert'. +* ``monitor_ssl_cert`` + Monitor SSL certificate, if monitor SSL is enabled and ``monitor_cert_source`` + is not 'cephadm-signed'. This should have the certificate .pem format. +* ``monitor_ssl_key`` + Monitor SSL key, if monitor SSL is enabled and ``monitor_cert_source`` is not + 'cephadm-signed'. This should have the key .pem format. +* ``monitor_ip_addrs`` + If ``monitor_ip_addrs`` is provided and the specified IP address is assigned to the host, + that IP address will be used. If IP address is not present, then 'monitor_networks' will be checked. +* ``monitor_networks`` + If ``monitor_networks`` is specified, an IP address that matches one of the specified + networks will be used. If IP not present, then default host ip will be used. .. _ingress-virtual-ip: diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 67257d8c67319..be17f4a4223a8 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -739,6 +739,8 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, self.cert_mgr.register_cert_key_pair('nvmeof', 'nvmeof_client_cert', 'nvmeof_client_key', TLSObjectScope.SERVICE) self.cert_mgr.register_cert('nvmeof', 'nvmeof_root_ca_cert', TLSObjectScope.SERVICE) + # register haproxy monitor ssl cert and key + self.cert_mgr.register_cert_key_pair('ingress', 'haproxy_monitor_ssl_cert', 'haproxy_monitor_ssl_key', TLSObjectScope.SERVICE) self.cert_mgr.init_tlsobject_store() diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index 5042f6ea27ae6..2f617501deac9 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -4,7 +4,7 @@ import random import string from typing import List, Dict, Any, Tuple, cast, Optional, TYPE_CHECKING -from ceph.deployment.service_spec import ServiceSpec, IngressSpec +from ceph.deployment.service_spec import ServiceSpec, IngressSpec, MonitorCertSource from mgr_util import build_url from cephadm import utils from orchestrator import OrchestratorError, DaemonDescription @@ -26,6 +26,14 @@ class IngressService(CephService): def needs_monitoring(self) -> bool: return True + @property + def haproxy_stats_cert_name(self) -> str: + return 'haproxy_monitor_ssl_cert' + + @property + def haproxy_stats_key_name(self) -> str: + return 'haproxy_monitor_ssl_key' + @classmethod def get_dependencies(cls, mgr: "CephadmOrchestrator", spec: Optional[ServiceSpec] = None, @@ -214,6 +222,23 @@ class IngressService(CephService): frontend_port = daemon_spec.ports[0] if daemon_spec.ports else spec.frontend_port if ip != '[::]' and frontend_port: daemon_spec.port_ips = {str(frontend_port): ip} + + monitor_ip, monitor_port = self.get_monitoring_details(daemon_spec.service_name, daemon_spec.host) + if monitor_ip: + monitor_ips = [monitor_ip] + daemon_spec.port_ips.update({str(monitor_port): monitor_ip}) + else: + monitor_ips = [ip, host_ip] + + monitor_ssl_file = None + cert_ips = [ip] + if spec.monitor_ssl: + if spec.monitor_cert_source == MonitorCertSource.REUSE_SERVICE_CERT.value: + monitor_ssl_file = 'haproxy.pem' + cert_ips.extend(monitor_ips) + else: + monitor_ssl_file = 'stats_haproxy.pem' + haproxy_conf = self.mgr.template.render( 'services/ingress/haproxy.cfg.j2', { @@ -224,12 +249,13 @@ class IngressService(CephService): 'user': spec.monitor_user or 'admin', 'password': password, 'ip': ip, + 'monitor_ips': monitor_ips, 'frontend_port': frontend_port, - 'monitor_port': daemon_spec.ports[1] if daemon_spec.ports else spec.monitor_port, - 'local_host_ip': host_ip, + 'monitor_port': spec.monitor_port, 'default_server_opts': server_opts, 'health_check_interval': spec.health_check_interval or '2s', 'v4v6_flag': v4v6_flag, + 'monitor_ssl_file': monitor_ssl_file, } ) config_files = { @@ -243,8 +269,30 @@ class IngressService(CephService): combined_pem = tls_pair.cert + '\n' + tls_pair.key config_files['files']['haproxy.pem'] = combined_pem + if spec.monitor_ssl and spec.monitor_cert_source != MonitorCertSource.REUSE_SERVICE_CERT.value: + stats_cert, stats_key = self.get_stats_certs(spec, daemon_spec, monitor_ips) + monitor_ssl_cert = [stats_cert, stats_key] + config_files['files']['stats_haproxy.pem'] = '\n'.join(monitor_ssl_cert) + return config_files, self.get_haproxy_dependencies(self.mgr, spec) + def get_stats_certs( + self, + svc_spec: IngressSpec, + daemon_spec: CephadmDaemonDeploySpec, + ips: Optional[List[str]] = None, + ) -> Tuple[str, str]: + return self.get_certificates_generic( + svc_spec=svc_spec, + daemon_spec=daemon_spec, + cert_attr='monitor_ssl_cert', + key_attr='monitor_ssl_key', + cert_source_attr='monitor_cert_source', + cert_name=self.haproxy_stats_cert_name, + key_name=self.haproxy_stats_key_name, + ips=ips + ) + def keepalived_prepare_create( self, daemon_spec: CephadmDaemonDeploySpec, @@ -445,3 +493,18 @@ class IngressService(CephService): } return config_file, self.get_keepalived_dependencies(self.mgr, spec) + + def get_monitoring_details(self, service_name: str, host: str) -> Tuple[Optional[str], Optional[int]]: + spec = cast(IngressSpec, self.mgr.spec_store[service_name].spec) + monitor_port = spec.monitor_port + + # check if monitor needs to be bind on specific ip + monitor_addr = spec.monitor_ip_addrs.get(host) if spec.monitor_ip_addrs else None + if monitor_addr and monitor_addr not in self.mgr.cache.get_host_network_ips(host): + logger.debug(f"Monitoring IP {monitor_addr} is not configured on host {host}.") + monitor_addr = None + if not monitor_addr and spec.monitor_networks: + monitor_addr = self.mgr.get_first_matching_network_ip(host, spec, spec.monitor_networks) + if not monitor_addr: + logger.debug(f"No IP address found in the network {spec.monitor_networks} on host {host}.") + return monitor_addr, monitor_port diff --git a/src/pybind/mgr/cephadm/services/service_discovery.py b/src/pybind/mgr/cephadm/services/service_discovery.py index 68c193e97de85..2c0478cb6684e 100644 --- a/src/pybind/mgr/cephadm/services/service_discovery.py +++ b/src/pybind/mgr/cephadm/services/service_discovery.py @@ -12,12 +12,12 @@ import orchestrator # noqa from mgr_util import build_url from typing import Dict, List, TYPE_CHECKING, cast, Collection, Callable, NamedTuple, Optional, IO from cephadm.services.nfs import NFSService +from cephadm.services.ingress import IngressService from cephadm.services.monitoring import AlertmanagerService, NodeExporterService, PrometheusService import secrets from mgr_util import verify_tls_files import tempfile -from cephadm.services.ingress import IngressSpec from cephadm.services.cephadmservice import CephExporterService from cephadm.services.nvmeof import NvmeofService from cephadm.services.service_registry import service_registry @@ -223,12 +223,13 @@ class Root(Server): srv_entries = [] for dd in self.mgr.cache.get_daemons_by_type('ingress'): if dd.service_name() in self.mgr.spec_store: - spec = cast(IngressSpec, self.mgr.spec_store[dd.service_name()].spec) assert dd.hostname is not None if dd.daemon_type == 'haproxy': - addr = self.mgr.inventory.get_addr(dd.hostname) + ingress = cast(IngressService, service_registry.get_service('ingress')) + monitor_ip, monitor_port = ingress.get_monitoring_details(dd.service_name(), dd.hostname) + addr = monitor_ip or dd.ip or self.mgr.inventory.get_addr(dd.hostname) srv_entries.append({ - 'targets': [f"{build_url(host=addr, port=spec.monitor_port).lstrip('/')}"], + 'targets': [f"{build_url(host=addr, port=monitor_port).lstrip('/')}"], 'labels': {'ingress': dd.service_name(), 'instance': dd.hostname} }) return srv_entries diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 index 9491685c4d188..943ed969ad7c2 100644 --- a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 +++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 @@ -45,10 +45,16 @@ defaults {% endif %} maxconn 8000 +{% if spec.enable_stats %} frontend stats mode http - bind {{ ip }}:{{ monitor_port }} - bind {{ local_host_ip }}:{{ monitor_port }} +{% for monitor_ip in monitor_ips %} + {% if spec.monitor_ssl %} + bind {{ monitor_ip }}:{{ monitor_port }} ssl crt /var/lib/haproxy/{{ monitor_ssl_file }} + {% else %} + bind {{ monitor_ip }}:{{ monitor_port }} + {% endif %} +{% endfor %} stats enable stats uri /stats stats refresh 10s @@ -56,6 +62,7 @@ frontend stats http-request use-service prometheus-exporter if { path /metrics } monitor-uri /health +{% endif %} frontend frontend {% if spec.ssl or spec.ssl_cert %} bind {{ ip }}:{{ frontend_port }} ssl crt /var/lib/haproxy/haproxy.pem {{ v4v6_flag }} diff --git a/src/pybind/mgr/cephadm/tests/test_service_discovery.py b/src/pybind/mgr/cephadm/tests/test_service_discovery.py index 32e20256626da..b560f81ce3192 100644 --- a/src/pybind/mgr/cephadm/tests/test_service_discovery.py +++ b/src/pybind/mgr/cephadm/tests/test_service_discovery.py @@ -68,6 +68,8 @@ class FakeNFSServiceSpec: class FakeIngressServiceSpec: def __init__(self, port): self.monitor_port = port + self.monitor_ip_addrs = {} + self.monitor_networks = {} class FakeServiceSpec: diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py index 0c44108056b81..902be51286e72 100644 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ b/src/pybind/mgr/cephadm/tests/test_services.py @@ -1274,7 +1274,8 @@ class TestMonitoring: monitor_password='12345', keepalived_password='12345', virtual_ip="1.2.3.4/32", - backend_service='rgw.foo')) as _, \ + backend_service='rgw.foo', + enable_stats=True)) as _, \ with_service(cephadm_module, NvmeofServiceSpec(service_id=f'{pool}.{group}', group=group, pool=pool)) as _, \ @@ -1437,7 +1438,8 @@ class TestMonitoring: monitor_password='12345', keepalived_password='12345', virtual_ip="1.2.3.4/32", - backend_service='rgw.foo')) as _, \ + backend_service='rgw.foo', + enable_stats=True)) as _, \ with_service(cephadm_module, NvmeofServiceSpec(service_id=f'{pool}.{group}', group=group, pool=pool)) as _, \ @@ -2735,6 +2737,7 @@ class TestIngressService: monitor_password='12345', keepalived_password='12345', enable_haproxy_protocol=enable_haproxy_protocol, + enable_stats=True ) cephadm_module.spec_store._specs = { @@ -2855,7 +2858,8 @@ class TestIngressService: monitor_password='12345', keepalived_password='12345', virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32") + virtual_ip="1.2.3.4/32", + enable_stats=True) with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: # generate the keepalived conf based on the specified spec keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( @@ -2984,7 +2988,8 @@ class TestIngressService: monitor_password='12345', keepalived_password='12345', virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32") + virtual_ip="1.2.3.4/32", + enable_stats=True) with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: # generate the keepalived conf based on the specified spec keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( @@ -3114,7 +3119,8 @@ class TestIngressService: monitor_password='12345', keepalived_password='12345', virtual_interface_networks=['1.2.3.0/24'], - virtual_ips_list=["1.2.3.4/32"]) + virtual_ip="1.2.3.4/32", + enable_stats=True) with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: # generate the keepalived conf based on the specified spec # Test with only 1 IP on the list, as it will fail with more VIPS but only one host. @@ -3196,7 +3202,7 @@ class TestIngressService: 'maxconn 8000\n' '\nfrontend stats\n ' 'mode http\n ' - 'bind [::]:8999\n ' + 'bind 1.2.3.4:8999\n ' 'bind 1.2.3.7:8999\n ' 'stats enable\n ' 'stats uri /stats\n ' @@ -3205,7 +3211,7 @@ class TestIngressService: 'http-request use-service prometheus-exporter if { path /metrics }\n ' 'monitor-uri /health\n' '\nfrontend frontend\n ' - 'bind [::]:8089 v4v6\n ' + 'bind 1.2.3.4:8089 \n ' 'default_backend backend\n\n' 'backend backend\n ' 'option forwardfor\n ' @@ -3253,7 +3259,8 @@ class TestIngressService: monitor_user='admin', monitor_password='12345', keepalived_password='12345', - virtual_ips_list=["1.2.3.100/24", "100.100.100.100/24"]) + virtual_ips_list=["1.2.3.100/24", "100.100.100.100/24"], + enable_stats=True) with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) @@ -3366,7 +3373,8 @@ class TestIngressService: monitor_user='admin', monitor_password='12345', keepalived_password='12345', - virtual_ips_list=["1.2.3.100/24", "100.100.100.100/24"]) + virtual_ips_list=["1.2.3.100/24", "100.100.100.100/24"], + enable_stats=True) with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: # since we're never actually going to refresh the host here, # check the tmp daemons to see what was placed during the apply @@ -3404,7 +3412,8 @@ class TestIngressService: monitor_user='admin', monitor_password='12345', keepalived_password='12345', - virtual_ip=f"{ip}/24") + virtual_ip=f"{ip}/24", + enable_stats=True) with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: # generate the haproxy conf based on the specified spec haproxy_daemon_spec = service_registry.get_service('ingress').prepare_create( @@ -3552,6 +3561,7 @@ class TestIngressService: monitor_password='12345', keepalived_password='12345', enable_haproxy_protocol=True, + enable_stats=True ) cephadm_module.spec_store._specs = { @@ -3929,6 +3939,355 @@ class TestNFS: ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] assert "Bind_addr = 1.2.3.7" in ganesha_conf + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_ingress_without_haproxy_stats(self, _run_cephadm, cephadm_module: CephadmOrchestrator): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + + with with_host(cephadm_module, 'test', addr='1.2.3.7'): + cephadm_module.cache.update_host_networks('test', { + '1.2.3.0/24': { + 'if0': [ + '1.2.3.4', # simulate already assigned VIP + '1.2.3.1', # simulate interface IP + ] + } + }) + + # the ingress backend + s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), + rgw_frontend_type='beast') + + ispec = IngressSpec(service_type='ingress', + service_id='test', + backend_service='rgw.foo', + frontend_port=8089, + monitor_port=8999, + monitor_user='admin', + monitor_password='12345', + keepalived_password='12345', + virtual_interface_networks=['1.2.3.0/24'], + virtual_ip="1.2.3.4/32") + with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: + # generate the haproxy conf based on the specified spec + haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( + CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) + + haproxy_expected_conf = { + 'files': + { + 'haproxy.cfg': + '# This file is generated by cephadm.' + '\nglobal\n log ' + '127.0.0.1 local2\n ' + 'chroot /var/lib/haproxy\n ' + 'pidfile /var/lib/haproxy/haproxy.pid\n ' + 'maxconn 8000\n ' + 'daemon\n ' + 'stats socket /var/lib/haproxy/stats\n' + '\ndefaults\n ' + 'mode http\n ' + 'log global\n ' + 'option httplog\n ' + 'option dontlognull\n ' + 'option http-server-close\n ' + 'option forwardfor except 127.0.0.0/8\n ' + 'option redispatch\n ' + 'retries 3\n ' + 'timeout queue 20s\n ' + 'timeout connect 5s\n ' + 'timeout http-request 1s\n ' + 'timeout http-keep-alive 5s\n ' + 'timeout client 30s\n ' + 'timeout server 30s\n ' + 'timeout check 5s\n ' + 'maxconn 8000\n' + '\nfrontend frontend\n ' + 'bind 1.2.3.4:8089\n ' + 'default_backend backend\n\n' + 'backend backend\n ' + 'option forwardfor\n ' + 'balance static-rr\n ' + 'option httpchk HEAD / HTTP/1.0\n ' + 'server ' + + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' + } + } + gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] + exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] + assert gen_config_lines == exp_config_lines + + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_ingress_haproxy_ssl(self, _run_cephadm, cephadm_module: CephadmOrchestrator): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + + with with_host(cephadm_module, 'test', addr='1.2.3.7'): + cephadm_module.cache.update_host_networks('test', { + '1.2.3.0/24': { + 'if0': [ + '1.2.3.4', # simulate already assigned VIP + '1.2.3.1', # simulate interface IP + ] + } + }) + + # the ingress backend + s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), + rgw_frontend_type='beast') + + ispec = IngressSpec(service_type='ingress', + service_id='test', + backend_service='rgw.foo', + frontend_port=8089, + monitor_port=8999, + monitor_user='admin', + monitor_password='12345', + keepalived_password='12345', + virtual_interface_networks=['1.2.3.0/24'], + virtual_ip="1.2.3.4/32", + ssl=True, + enable_stats=True, + monitor_ssl=True) + with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: + # generate the haproxy conf based on the specified spec + haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( + CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) + + haproxy_expected_conf = { + 'files': + { + 'haproxy.cfg': + '# This file is generated by cephadm.' + '\nglobal\n log ' + '127.0.0.1 local2\n ' + 'chroot /var/lib/haproxy\n ' + 'pidfile /var/lib/haproxy/haproxy.pid\n ' + 'maxconn 8000\n ' + 'daemon\n ' + 'stats socket /var/lib/haproxy/stats\n' + '\ndefaults\n ' + 'mode http\n ' + 'log global\n ' + 'option httplog\n ' + 'option dontlognull\n ' + 'option http-server-close\n ' + 'option forwardfor except 127.0.0.0/8\n ' + 'option redispatch\n ' + 'retries 3\n ' + 'timeout queue 20s\n ' + 'timeout connect 5s\n ' + 'timeout http-request 1s\n ' + 'timeout http-keep-alive 5s\n ' + 'timeout client 30s\n ' + 'timeout server 30s\n ' + 'timeout check 5s\n ' + 'maxconn 8000\n' + '\nfrontend stats\n ' + 'mode http\n ' + 'bind 1.2.3.4:8999 ssl crt /var/lib/haproxy/haproxy.pem\n ' + 'bind 1.2.3.7:8999 ssl crt /var/lib/haproxy/haproxy.pem\n ' + 'stats enable\n ' + 'stats uri /stats\n ' + 'stats refresh 10s\n ' + 'stats auth admin:12345\n ' + 'http-request use-service prometheus-exporter if { path /metrics }\n ' + 'monitor-uri /health\n' + '\nfrontend frontend\n ' + 'bind 1.2.3.4:8089 ssl crt /var/lib/haproxy/haproxy.pem\n ' + 'default_backend backend\n\n' + 'backend backend\n ' + 'option forwardfor\n ' + 'balance static-rr\n ' + 'option httpchk HEAD / HTTP/1.0\n ' + 'server ' + + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' + } + } + gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] + exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] + assert gen_config_lines == exp_config_lines + + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_ingress_haproxy_with_different_stats_cert(self, _run_cephadm, cephadm_module: CephadmOrchestrator): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + + with with_host(cephadm_module, 'test', addr='1.2.3.7'): + cephadm_module.cache.update_host_networks('test', { + '1.2.3.0/24': { + 'if0': [ + '1.2.3.4', # simulate already assigned VIP + '1.2.3.1', # simulate interface IP + ] + } + }) + + # the ingress backend + s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), + rgw_frontend_type='beast') + + ispec = IngressSpec(service_type='ingress', + service_id='test', + backend_service='rgw.foo', + frontend_port=8089, + monitor_port=8999, + monitor_user='admin', + monitor_password='12345', + keepalived_password='12345', + virtual_interface_networks=['1.2.3.0/24'], + virtual_ip="1.2.3.4/32", + ssl=True, + enable_stats=True, + monitor_ssl=True, + monitor_cert_source='cephadm-signed' + ) + with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: + # generate the haproxy conf based on the specified spec + haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( + CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) + + haproxy_expected_conf = { + 'files': + { + 'haproxy.cfg': + '# This file is generated by cephadm.' + '\nglobal\n log ' + '127.0.0.1 local2\n ' + 'chroot /var/lib/haproxy\n ' + 'pidfile /var/lib/haproxy/haproxy.pid\n ' + 'maxconn 8000\n ' + 'daemon\n ' + 'stats socket /var/lib/haproxy/stats\n' + '\ndefaults\n ' + 'mode http\n ' + 'log global\n ' + 'option httplog\n ' + 'option dontlognull\n ' + 'option http-server-close\n ' + 'option forwardfor except 127.0.0.0/8\n ' + 'option redispatch\n ' + 'retries 3\n ' + 'timeout queue 20s\n ' + 'timeout connect 5s\n ' + 'timeout http-request 1s\n ' + 'timeout http-keep-alive 5s\n ' + 'timeout client 30s\n ' + 'timeout server 30s\n ' + 'timeout check 5s\n ' + 'maxconn 8000\n' + '\nfrontend stats\n ' + 'mode http\n ' + 'bind 1.2.3.4:8999 ssl crt /var/lib/haproxy/stats_haproxy.pem\n ' + 'bind 1.2.3.7:8999 ssl crt /var/lib/haproxy/stats_haproxy.pem\n ' + 'stats enable\n ' + 'stats uri /stats\n ' + 'stats refresh 10s\n ' + 'stats auth admin:12345\n ' + 'http-request use-service prometheus-exporter if { path /metrics }\n ' + 'monitor-uri /health\n' + '\nfrontend frontend\n ' + 'bind 1.2.3.4:8089 ssl crt /var/lib/haproxy/haproxy.pem\n ' + 'default_backend backend\n\n' + 'backend backend\n ' + 'option forwardfor\n ' + 'balance static-rr\n ' + 'option httpchk HEAD / HTTP/1.0\n ' + 'server ' + + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' + } + } + gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] + exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] + assert gen_config_lines == exp_config_lines + + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_ingress_haproxy_monitor_ip(self, _run_cephadm, cephadm_module: CephadmOrchestrator): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + + with with_host(cephadm_module, 'test', addr='1.2.3.7'): + cephadm_module.cache.update_host_networks('test', { + '1.2.3.0/24': { + 'if0': [ + '1.2.3.4', # simulate already assigned VIP + '1.2.3.1', # simulate interface IP + ] + } + }) + + # the ingress backend + s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), + rgw_frontend_type='beast') + + ispec = IngressSpec(service_type='ingress', + service_id='test', + backend_service='rgw.foo', + frontend_port=8089, + monitor_port=8999, + monitor_user='admin', + monitor_password='12345', + keepalived_password='12345', + virtual_interface_networks=['1.2.3.0/24'], + virtual_ip="1.2.3.4/32", + enable_stats=True, + monitor_ip_addrs={'test': '1.2.3.1'}) + with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: + # generate the haproxy conf based on the specified spec + haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( + CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) + + haproxy_expected_conf = { + 'files': + { + 'haproxy.cfg': + '# This file is generated by cephadm.' + '\nglobal\n log ' + '127.0.0.1 local2\n ' + 'chroot /var/lib/haproxy\n ' + 'pidfile /var/lib/haproxy/haproxy.pid\n ' + 'maxconn 8000\n ' + 'daemon\n ' + 'stats socket /var/lib/haproxy/stats\n' + '\ndefaults\n ' + 'mode http\n ' + 'log global\n ' + 'option httplog\n ' + 'option dontlognull\n ' + 'option http-server-close\n ' + 'option forwardfor except 127.0.0.0/8\n ' + 'option redispatch\n ' + 'retries 3\n ' + 'timeout queue 20s\n ' + 'timeout connect 5s\n ' + 'timeout http-request 1s\n ' + 'timeout http-keep-alive 5s\n ' + 'timeout client 30s\n ' + 'timeout server 30s\n ' + 'timeout check 5s\n ' + 'maxconn 8000\n' + '\nfrontend stats\n ' + 'mode http\n ' + 'bind 1.2.3.1:8999\n ' + 'stats enable\n ' + 'stats uri /stats\n ' + 'stats refresh 10s\n ' + 'stats auth admin:12345\n ' + 'http-request use-service prometheus-exporter if { path /metrics }\n ' + 'monitor-uri /health\n' + '\nfrontend frontend\n ' + 'bind 1.2.3.4:8089\n ' + 'default_backend backend\n\n' + 'backend backend\n ' + 'option forwardfor\n ' + 'balance static-rr\n ' + 'option httpchk HEAD / HTTP/1.0\n ' + 'server ' + + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' + } + } + + gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] + exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] + + assert gen_config_lines == exp_config_lines + class TestCephFsMirror: @patch("cephadm.serve.CephadmServe._run_cephadm") diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 10b1d391f8f11..510e0028d01e1 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -71,6 +71,16 @@ class CertificateSource(Enum): CEPHADM_SIGNED = "cephadm-signed" +class MonitorCertSource(Enum): + """ + - REUSE_SERVICE_CERT: Use Service cert for monitoring + """ + INLINE = CertificateSource.INLINE.value + REFERENCE = CertificateSource.REFERENCE.value + CEPHADM_SIGNED = CertificateSource.CEPHADM_SIGNED.value + REUSE_SERVICE_CERT = "reuse_service_cert" + + def handle_type_error(method: FuncT) -> FuncT: @wraps(method) def inner(cls: Any, *args: Any, **kwargs: Any) -> Any: @@ -2151,6 +2161,12 @@ class IngressSpec(ServiceSpec): extra_entrypoint_args: Optional[GeneralArgList] = None, custom_configs: Optional[List[CustomConfig]] = None, health_check_interval: Optional[str] = None, + monitor_ssl: bool = False, + monitor_ssl_cert: Optional[str] = None, + monitor_ssl_key: Optional[str] = None, + monitor_cert_source: Optional[str] = MonitorCertSource.REUSE_SERVICE_CERT.value, + monitor_networks: Optional[List[str]] = None, + monitor_ip_addrs: Optional[Dict[str, str]] = None, ): assert service_type == 'ingress' @@ -2188,6 +2204,13 @@ class IngressSpec(ServiceSpec): self.enable_haproxy_protocol = enable_haproxy_protocol self.health_check_interval = health_check_interval.strip( ) if health_check_interval else None + self.enable_stats = enable_stats + self.monitor_ssl = monitor_ssl + self.monitor_ssl_cert = monitor_ssl_cert + self.monitor_ssl_key = monitor_ssl_key + self.monitor_cert_source = monitor_cert_source + self.monitor_networks = monitor_networks + self.monitor_ip_addrs = monitor_ip_addrs def get_port_start(self) -> List[int]: ports = [] @@ -2226,6 +2249,18 @@ class IngressSpec(ServiceSpec): f'Cannot add ingress: Invalid health_check_interval specified. ' f'Valid units are: {valid_units}') + # validate SSL parametes + if self.monitor_ssl: + if not self.ssl: + raise SpecValidationError( + 'To enable SSL for stats, SSL must also be enabled on the frontend.' + ) + if self.monitor_ssl_cert and bool(self.monitor_ssl_cert) != bool(self.monitor_ssl_key): + raise SpecValidationError( + 'To enable monitor_ssl, both monitor_ssl_cert and monitor_ssl_key ' + 'must be provided.' + ) + yaml.add_representer(IngressSpec, ServiceSpec.yaml_representer) diff --git a/src/python-common/ceph/tests/test_service_spec.py b/src/python-common/ceph/tests/test_service_spec.py index 89b608f1314aa..a855406fc7374 100644 --- a/src/python-common/ceph/tests/test_service_spec.py +++ b/src/python-common/ceph/tests/test_service_spec.py @@ -398,6 +398,7 @@ spec: certificate_source: cephadm-signed first_virtual_router_id: 50 frontend_port: 8080 + monitor_cert_source: reuse_service_cert monitor_port: 8081 virtual_ip: 192.168.20.1/24 ---