From: Redouane Kachach Date: Thu, 26 Feb 2026 17:25:54 +0000 (+0100) Subject: mgr/cephadm: removed big test_services file X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2d12df2f69099edd9b624ef433a359a398c2528d;p=ceph.git mgr/cephadm: removed big test_services file Signed-off-by: Redouane Kachach --- diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py deleted file mode 100644 index 055fad07f85..00000000000 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ /dev/null @@ -1,5766 +0,0 @@ -from textwrap import dedent -import json -import urllib.parse -import yaml -from mgr_util import build_url - -import pytest - -from unittest.mock import Mock, MagicMock, call, patch, ANY - -from cephadm.serve import CephadmServe -from cephadm.services.service_registry import service_registry -from cephadm.services.cephadmservice import MonService, CephadmDaemonDeploySpec -from cephadm.services.iscsi import IscsiService -from cephadm.services.nvmeof import NvmeofService -from cephadm.services.monitoring import GrafanaService, AlertmanagerService, PrometheusService -from cephadm.services.smb import SMBSpec -from cephadm.module import CephadmOrchestrator -from ceph.deployment.service_spec import ( - AlertManagerSpec, - CephExporterSpec, - CustomContainerSpec, - GrafanaSpec, - IngressSpec, - IscsiServiceSpec, - MonitoringSpec, - NFSServiceSpec, - NvmeofServiceSpec, - PlacementSpec, - PrometheusSpec, - RGWSpec, - SNMPGatewaySpec, - ServiceSpec, - TracingSpec, - MgmtGatewaySpec, - OAuth2ProxySpec -) -from cephadm.tests.fixtures import with_host, with_service, _run_cephadm, async_side_effect, wait -from cephadm.tlsobject_types import TLSCredentials - -from ceph.utils import datetime_now - -from orchestrator import OrchestratorError -from orchestrator._interface import DaemonDescription - -from typing import Dict, List - -cephadm_root_ca = """-----BEGIN CERTIFICATE-----\nMIIE7DCCAtSgAwIBAgIUE8b2zZ64geu2ns3Zfn3/4L+Cf6MwDQYJKoZIhvcNAQEL\nBQAwFzEVMBMGA1UEAwwMY2VwaGFkbS1yb290MB4XDTI0MDYyNjE0NDA1M1oXDTM0\nMDYyNzE0NDA1M1owFzEVMBMGA1UEAwwMY2VwaGFkbS1yb290MIICIjANBgkqhkiG\n9w0BAQEFAAOCAg8AMIICCgKCAgEAsZRJsdtTr9GLG1lWFql5SGc46ldFanNJd1Gl\nqXq5vgZVKRDTmNgAb/XFuNEEmbDAXYIRZolZeYKMHfn0pouPRSel0OsC6/02ZUOW\nIuN89Wgo3IYleCFpkVIumD8URP3hwdu85plRxYZTtlruBaTRH38lssyCqxaOdEt7\nAUhvYhcMPJThB17eOSQ73mb8JEC83vB47fosI7IhZuvXvRSuZwUW30rJanWNhyZq\neS2B8qw2RSO0+77H6gA4ftBnitfsE1Y8/F9Z/f92JOZuSMQXUB07msznPbRJia3f\nueO8gOc32vxd1A1/Qzp14uX34yEGY9ko2lW226cZO29IVUtXOX+LueQttwtdlpz8\ne6Npm09pXhXAHxV/OW3M28MdXmobIqT/m9MfkeAErt5guUeC5y8doz6/3VQRjFEn\nRpN0WkblgnNAQ3DONPc+Qd9Fi/wZV2X7bXoYpNdoWDsEOiE/eLmhG1A2GqU/mneP\nzQ6u79nbdwTYpwqHpa+PvusXeLfKauzI8lLUJotdXy9EK8iHUofibB61OljYye6B\nG3b8C4QfGsw8cDb4APZd/6AZYyMx/V3cGZ+GcOV7WvsC8k7yx5Uqasm/kiGQ3EZo\nuNenNEYoGYrjb8D/8QzqNUTwlEh27/ps80tO7l2GGTvWVZL0PRZbmLDvO77amtOf\nOiRXMoUCAwEAAaMwMC4wGwYDVR0RBBQwEocQAAAAAAAAAAAAAAAAAAAAATAPBgNV\nHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAxwzX5AhYEWhTV4VUwUj5+\nqPdl4Q2tIxRokqyE+cDxoSd+6JfGUefUbNyBxDt0HaBq8obDqqrbcytxnn7mpnDu\nhtiauY+I4Amt7hqFOiFA4cCLi2mfok6g2vL53tvhd9IrsfflAU2wy7hL76Ejm5El\nA+nXlkJwps01Whl9pBkUvIbOn3pXX50LT4hb5zN0PSu957rjd2xb4HdfuySm6nW4\n4GxtVWfmGA6zbC4XMEwvkuhZ7kD2qjkAguGDF01uMglkrkCJT3OROlNBuSTSBGqt\ntntp5VytHvb7KTF7GttM3ha8/EU2KYaHM6WImQQTrOfiImAktOk4B3lzUZX3HYIx\n+sByO4P4dCvAoGz1nlWYB2AvCOGbKf0Tgrh4t4jkiF8FHTXGdfvWmjgi1pddCNAy\nn65WOCmVmLZPERAHOk1oBwqyReSvgoCFo8FxbZcNxJdlhM0Z6hzKggm3O3Dl88Xl\n5euqJjh2STkBW8Xuowkg1TOs5XyWvKoDFAUzyzeLOL8YSG+gXV22gPTUaPSVAqdb\nwd0Fx2kjConuC5bgTzQHs8XWA930U3XWZraj21Vaa8UxlBLH4fUro8H5lMSYlZNE\nJHRNW8BkznAClaFSDG3dybLsrzrBFAu/Qb5zVkT1xyq0YkepGB7leXwq6vjWA5Pw\nmZbKSphWfh0qipoqxqhfkw==\n-----END CERTIFICATE-----\n""" - -ceph_generated_cert = """-----BEGIN CERTIFICATE-----\nMIICxjCCAa4CEQDIZSujNBlKaLJzmvntjukjMA0GCSqGSIb3DQEBDQUAMCExDTAL\nBgNVBAoMBENlcGgxEDAOBgNVBAMMB2NlcGhhZG0wHhcNMjIwNzEzMTE0NzA3WhcN\nMzIwNzEwMTE0NzA3WjAhMQ0wCwYDVQQKDARDZXBoMRAwDgYDVQQDDAdjZXBoYWRt\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyyMe4DMA+MeYK7BHZMHB\nq7zjliEOcNgxomjU8qbf5USF7Mqrf6+/87XWqj4pCyAW8x0WXEr6A56a+cmBVmt+\nqtWDzl020aoId6lL5EgLLn6/kMDCCJLq++Lg9cEofMSvcZh+lY2f+1p+C+00xent\nrLXvXGOilAZWaQfojT2BpRnNWWIFbpFwlcKrlg2G0cFjV5c1m6a0wpsQ9JHOieq0\nSvwCixajwq3CwAYuuiU1wjI4oJO4Io1+g8yB3nH2Mo/25SApCxMXuXh4kHLQr/T4\n4hqisvG4uJYgKMcSIrWj5o25mclByGi1UI/kZkCUES94i7Z/3ihx4Bad0AMs/9tw\nFwIDAQABMA0GCSqGSIb3DQEBDQUAA4IBAQAf+pwz7Gd7mDwU2LY0TQXsK6/8KGzh\nHuX+ErOb8h5cOAbvCnHjyJFWf6gCITG98k9nxU9NToG0WYuNm/max1y/54f0dtxZ\npUo6KSNl3w6iYCfGOeUIj8isi06xMmeTgMNzv8DYhDt+P2igN6LenqWTVztogkiV\nxQ5ZJFFLEw4sN0CXnrZX3t5ruakxLXLTLKeE0I91YJvjClSBGkVJq26wOKQNHMhx\npWxeydQ5EgPZY+Aviz5Dnxe8aB7oSSovpXByzxURSabOuCK21awW5WJCGNpmqhWK\nZzACBDEstccj57c4OGV0eayHJRsluVr2e9NHRINZA3qdB37e6gsI1xHo\n-----END CERTIFICATE-----\n""" - -ceph_generated_key = """-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDLIx7gMwD4x5gr\nsEdkwcGrvOOWIQ5w2DGiaNTypt/lRIXsyqt/r7/ztdaqPikLIBbzHRZcSvoDnpr5\nyYFWa36q1YPOXTbRqgh3qUvkSAsufr+QwMIIkur74uD1wSh8xK9xmH6VjZ/7Wn4L\n7TTF6e2ste9cY6KUBlZpB+iNPYGlGc1ZYgVukXCVwquWDYbRwWNXlzWbprTCmxD0\nkc6J6rRK/AKLFqPCrcLABi66JTXCMjigk7gijX6DzIHecfYyj/blICkLExe5eHiQ\nctCv9PjiGqKy8bi4liAoxxIitaPmjbmZyUHIaLVQj+RmQJQRL3iLtn/eKHHgFp3Q\nAyz/23AXAgMBAAECggEAVoTB3Mm8azlPlaQB9GcV3tiXslSn+uYJ1duCf0sV52dV\nBzKW8s5fGiTjpiTNhGCJhchowqxoaew+o47wmGc2TvqbpeRLuecKrjScD0GkCYyQ\neM2wlshEbz4FhIZdgS6gbuh9WaM1dW/oaZoBNR5aTYo7xYTmNNeyLA/jO2zr7+4W\n5yES1lMSBXpKk7bDGKYY4bsX2b5RLr2Grh2u2bp7hoLABCEvuu8tSQdWXLEXWpXo\njwmV3hc6tabypIa0mj2Dmn2Dmt1ppSO0AZWG/WAizN3f4Z0r/u9HnbVrVmh0IEDw\n3uf2LP5o3msG9qKCbzv3lMgt9mMr70HOKnJ8ohMSKQKBgQDLkNb+0nr152HU9AeJ\nvdz8BeMxcwxCG77iwZphZ1HprmYKvvXgedqWtS6FRU+nV6UuQoPUbQxJBQzrN1Qv\nwKSlOAPCrTJgNgF/RbfxZTrIgCPuK2KM8I89VZv92TSGi362oQA4MazXC8RAWjoJ\nSu1/PHzK3aXOfVNSLrOWvIYeZQKBgQD/dgT6RUXKg0UhmXj7ExevV+c7oOJTDlMl\nvLngrmbjRgPO9VxLnZQGdyaBJeRngU/UXfNgajT/MU8B5fSKInnTMawv/tW7634B\nw3v6n5kNIMIjJmENRsXBVMllDTkT9S7ApV+VoGnXRccbTiDapBThSGd0wri/CuwK\nNWK1YFOeywKBgEDyI/XG114PBUJ43NLQVWm+wx5qszWAPqV/2S5MVXD1qC6zgCSv\nG9NLWN1CIMimCNg6dm7Wn73IM7fzvhNCJgVkWqbItTLG6DFf3/DPODLx1wTMqLOI\nqFqMLqmNm9l1Nec0dKp5BsjRQzq4zp1aX21hsfrTPmwjxeqJZdioqy2VAoGAXR5X\nCCdSHlSlUW8RE2xNOOQw7KJjfWT+WAYoN0c7R+MQplL31rRU7dpm1bLLRBN11vJ8\nMYvlT5RYuVdqQSP6BkrX+hLJNBvOLbRlL+EXOBrVyVxHCkDe+u7+DnC4epbn+N8P\nLYpwqkDMKB7diPVAizIKTBxinXjMu5fkKDs5n+sCgYBbZheYKk5M0sIxiDfZuXGB\nkf4mJdEkTI1KUGRdCwO/O7hXbroGoUVJTwqBLi1tKqLLarwCITje2T200BYOzj82\nqwRkCXGtXPKnxYEEUOiFx9OeDrzsZV00cxsEnX0Zdj+PucQ/J3Cvd0dWUspJfLHJ\n39gnaegswnz9KMQAvzKFdg==\n-----END PRIVATE KEY-----\n""" - - -class FakeInventory: - def get_addr(self, name: str) -> str: - return '1.2.3.4' - - -class FakeMgr: - def __init__(self): - self.config = '' - self.set_mon_crush_locations: Dict[str, List[str]] = {} - self.check_mon_command = MagicMock(side_effect=self._check_mon_command) - self.mon_command = MagicMock(side_effect=self._check_mon_command) - self.template = MagicMock() - self.log = MagicMock() - self.cert_mgr = MagicMock() - self.inventory = FakeInventory() - - def _check_mon_command(self, cmd_dict, inbuf=None): - prefix = cmd_dict.get('prefix') - if prefix == 'get-cmd': - return 0, self.config, '' - if prefix == 'set-cmd': - self.config = cmd_dict.get('value') - return 0, 'value set', '' - if prefix in ['auth get']: - return 0, '[foo]\nkeyring = asdf\n', '' - if prefix == 'quorum_status': - # actual quorum status output from testing - # note in this output all of the mons have blank crush locations - return 0, """{"election_epoch": 14, "quorum": [0, 1, 2], "quorum_names": ["vm-00", "vm-01", "vm-02"], "quorum_leader_name": "vm-00", "quorum_age": 101, "features": {"quorum_con": "4540138322906710015", "quorum_mon": ["kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging", "quincy", "reef"]}, "monmap": {"epoch": 3, "fsid": "9863e1b8-6f24-11ed-8ad8-525400c13ad2", "modified": "2022-11-28T14:00:29.972488Z", "created": "2022-11-28T13:57:55.847497Z", "min_mon_release": 18, "min_mon_release_name": "reef", "election_strategy": 1, "disallowed_leaders: ": "", "stretch_mode": false, "tiebreaker_mon": "", "features": {"persistent": ["kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging", "quincy", "reef"], "optional": []}, "mons": [{"rank": 0, "name": "vm-00", "public_addrs": {"addrvec": [{"type": "v2", "addr": "192.168.122.61:3300", "nonce": 0}, {"type": "v1", "addr": "192.168.122.61:6789", "nonce": 0}]}, "addr": "192.168.122.61:6789/0", "public_addr": "192.168.122.61:6789/0", "priority": 0, "weight": 0, "crush_location": "{}"}, {"rank": 1, "name": "vm-01", "public_addrs": {"addrvec": [{"type": "v2", "addr": "192.168.122.63:3300", "nonce": 0}, {"type": "v1", "addr": "192.168.122.63:6789", "nonce": 0}]}, "addr": "192.168.122.63:6789/0", "public_addr": "192.168.122.63:6789/0", "priority": 0, "weight": 0, "crush_location": "{}"}, {"rank": 2, "name": "vm-02", "public_addrs": {"addrvec": [{"type": "v2", "addr": "192.168.122.82:3300", "nonce": 0}, {"type": "v1", "addr": "192.168.122.82:6789", "nonce": 0}]}, "addr": "192.168.122.82:6789/0", "public_addr": "192.168.122.82:6789/0", "priority": 0, "weight": 0, "crush_location": "{}"}]}}""", '' - if prefix == 'mon set_location': - self.set_mon_crush_locations[cmd_dict.get('name')] = cmd_dict.get('args') - return 0, '', '' - return -1, '', 'error' - - def get_minimal_ceph_conf(self) -> str: - return '' - - def get_mgr_ip(self) -> str: - return '1.2.3.4' - - -class TestCephadmService: - def test_set_value_on_dashboard(self): - # pylint: disable=protected-access - mgr = FakeMgr() - service_url = 'http://svc:1000' - service = GrafanaService(mgr) - service._set_value_on_dashboard('svc', 'get-cmd', 'set-cmd', service_url) - assert mgr.config == service_url - - # set-cmd should not be called if value doesn't change - mgr.check_mon_command.reset_mock() - service._set_value_on_dashboard('svc', 'get-cmd', 'set-cmd', service_url) - mgr.check_mon_command.assert_called_once_with({'prefix': 'get-cmd'}) - - def test_get_auth_entity(self): - mgr = FakeMgr() - service_registry.init_services(mgr) - - for daemon_type in ['rgw', 'rbd-mirror', 'nfs', "iscsi"]: - assert "client.%s.id1" % (daemon_type) == \ - service_registry.get_service(daemon_type).get_auth_entity("id1", "host") - assert "client.%s.id1" % (daemon_type) == \ - service_registry.get_service(daemon_type).get_auth_entity("id1", "") - assert "client.%s.id1" % (daemon_type) == \ - service_registry.get_service(daemon_type).get_auth_entity("id1") - - assert "client.crash.host" == \ - service_registry.get_service('crash').get_auth_entity("id1", "host") - with pytest.raises(OrchestratorError): - service_registry.get_service('crash').get_auth_entity("id1", "") - service_registry.get_service('crash').get_auth_entity("id1") - - assert "mon." == service_registry.get_service('mon').get_auth_entity("id1", "host") - assert "mon." == service_registry.get_service('mon').get_auth_entity("id1", "") - assert "mon." == service_registry.get_service('mon').get_auth_entity("id1") - - assert "mgr.id1" == service_registry.get_service('mgr').get_auth_entity("id1", "host") - assert "mgr.id1" == service_registry.get_service('mgr').get_auth_entity("id1", "") - assert "mgr.id1" == service_registry.get_service('mgr').get_auth_entity("id1") - - for daemon_type in ["osd", "mds"]: - assert "%s.id1" % daemon_type == \ - service_registry.get_service(daemon_type).get_auth_entity("id1", "host") - assert "%s.id1" % daemon_type == \ - service_registry.get_service(daemon_type).get_auth_entity("id1", "") - assert "%s.id1" % daemon_type == \ - service_registry.get_service(daemon_type).get_auth_entity("id1") - - # services based on CephadmService shouldn't have get_auth_entity - with pytest.raises(AttributeError): - for daemon_type in ['grafana', 'alertmanager', 'prometheus', 'node-exporter', 'loki', 'promtail', 'alloy']: - service_registry.get_service(daemon_type).get_auth_entity("id1", "host") - service_registry.get_service(daemon_type).get_auth_entity("id1", "") - service_registry.get_service(daemon_type).get_auth_entity("id1") - - -class TestISCSIService: - - mgr = FakeMgr() - iscsi_service = IscsiService(mgr) - - iscsi_spec = IscsiServiceSpec(service_type='iscsi', service_id="a") - iscsi_spec.daemon_type = "iscsi" - iscsi_spec.daemon_id = "a" - iscsi_spec.spec = MagicMock() - iscsi_spec.spec.daemon_type = "iscsi" - iscsi_spec.spec.ssl_cert = '' - iscsi_spec.api_user = "user" - iscsi_spec.api_password = "password" - iscsi_spec.api_port = 5000 - iscsi_spec.api_secure = False - iscsi_spec.ssl_cert = "cert" - iscsi_spec.ssl_key = "key" - - mgr.spec_store = MagicMock() - mgr.spec_store.all_specs.get.return_value = iscsi_spec - - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - def test_iscsi_client_caps(self): - - iscsi_daemon_spec = CephadmDaemonDeploySpec( - host='host', daemon_id='a', service_name=self.iscsi_spec.service_name()) - - self.iscsi_service.prepare_create(iscsi_daemon_spec) - - expected_caps = ['mon', - 'profile rbd, allow command "osd blocklist", allow command "config-key get" with "key" prefix "iscsi/"', - 'mgr', 'allow command "service status"', - 'osd', 'allow rwx'] - - expected_call = call({'prefix': 'auth get-or-create', - 'entity': 'client.iscsi.a', - 'caps': expected_caps}) - expected_call2 = call({'prefix': 'auth caps', - 'entity': 'client.iscsi.a', - 'caps': expected_caps}) - expected_call3 = call({'prefix': 'auth get', - 'entity': 'client.iscsi.a'}) - - assert expected_call in self.mgr.mon_command.mock_calls - assert expected_call2 in self.mgr.mon_command.mock_calls - assert expected_call3 in self.mgr.mon_command.mock_calls - - @patch('cephadm.utils.resolve_ip') - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - def test_iscsi_dashboard_config(self, mock_resolve_ip): - - self.mgr.check_mon_command = MagicMock() - self.mgr.check_mon_command.return_value = ('', '{"gateways": {}}', '') - - # Case 1: use IPV4 address - id1 = DaemonDescription(daemon_type='iscsi', hostname="testhost1", - daemon_id="a", ip='192.168.1.1') - daemon_list = [id1] - mock_resolve_ip.return_value = '192.168.1.1' - - self.iscsi_service.config_dashboard(daemon_list) - - dashboard_expected_call = call({'prefix': 'dashboard iscsi-gateway-add', - 'name': 'testhost1'}, - 'http://user:password@192.168.1.1:5000') - - assert dashboard_expected_call in self.mgr.check_mon_command.mock_calls - - # Case 2: use IPV6 address - self.mgr.check_mon_command.reset_mock() - - id1 = DaemonDescription(daemon_type='iscsi', hostname="testhost1", - daemon_id="a", ip='FEDC:BA98:7654:3210:FEDC:BA98:7654:3210') - mock_resolve_ip.return_value = 'FEDC:BA98:7654:3210:FEDC:BA98:7654:3210' - - self.iscsi_service.config_dashboard(daemon_list) - - dashboard_expected_call = call({'prefix': 'dashboard iscsi-gateway-add', - 'name': 'testhost1'}, - 'http://user:password@[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:5000') - - assert dashboard_expected_call in self.mgr.check_mon_command.mock_calls - - # Case 3: IPV6 Address . Secure protocol - self.mgr.check_mon_command.reset_mock() - - self.iscsi_spec.api_secure = True - - self.iscsi_service.config_dashboard(daemon_list) - - dashboard_expected_call = call({'prefix': 'dashboard iscsi-gateway-add', - 'name': 'testhost1'}, - 'https://user:password@[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:5000') - - assert dashboard_expected_call in self.mgr.check_mon_command.mock_calls - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator.get_unique_name") - @patch("cephadm.services.iscsi.get_trusted_ips") - def test_iscsi_config(self, _get_trusted_ips, _get_name, _run_cephadm, cephadm_module: CephadmOrchestrator): - - iscsi_daemon_id = 'testpool.test.qwert' - trusted_ips = '1.1.1.1,2.2.2.2' - api_port = 3456 - api_user = 'test-user' - api_password = 'test-password' - pool = 'testpool' - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - _get_name.return_value = iscsi_daemon_id - _get_trusted_ips.return_value = trusted_ips - - iscsi_gateway_conf = f"""# This file is generated by cephadm. -[config] -cluster_client_name = client.iscsi.{iscsi_daemon_id} -pool = {pool} -trusted_ip_list = {trusted_ips} -minimum_gateways = 1 -api_port = {api_port} -api_user = {api_user} -api_password = {api_password} -api_secure = False -log_to_stderr = True -log_to_stderr_prefix = debug -log_to_file = False""" - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, IscsiServiceSpec(service_id=pool, - api_port=api_port, - api_user=api_user, - api_password=api_password, - pool=pool, - trusted_ip_list=trusted_ips)): - _run_cephadm.assert_called_with( - 'test', - f'iscsi.{iscsi_daemon_id}', - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": f'iscsi.{iscsi_daemon_id}', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [api_port], - }, - "meta": { - 'service_name': f'iscsi.{pool}', - 'ports': [api_port], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "config": "", - "keyring": f"[client.iscsi.{iscsi_daemon_id}]\nkey = None\n", - "files": { - "iscsi-gateway.cfg": iscsi_gateway_conf, - }, - } - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator.get_unique_name") - @patch("cephadm.services.iscsi.get_trusted_ips") - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - def test_iscsi_config_with_security_enabled(self, _get_trusted_ips, _get_name, _run_cephadm, cephadm_module: CephadmOrchestrator): - - iscsi_daemon_id = 'testpool.test.qwert' - trusted_ips = '1.1.1.1,2.2.2.2' - api_port = 3456 - api_user = 'test-user' - api_password = 'test-password' - pool = 'testpool' - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - _get_name.return_value = iscsi_daemon_id - _get_trusted_ips.return_value = trusted_ips - - cephadm_module.check_mon_command = MagicMock() - cephadm_module.check_mon_command.return_value = (0, '', '') - - iscsi_gateway_conf = f"""# This file is generated by cephadm. -[config] -cluster_client_name = client.iscsi.{iscsi_daemon_id} -pool = {pool} -trusted_ip_list = {trusted_ips} -minimum_gateways = 1 -api_port = {api_port} -api_user = {api_user} -api_password = {api_password} -api_secure = True -log_to_stderr = True -log_to_stderr_prefix = debug -log_to_file = False""" - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, IscsiServiceSpec(service_id=pool, - api_port=api_port, - api_user=api_user, - api_password=api_password, - pool=pool, - api_secure=True, - ssl_cert=ceph_generated_cert, - ssl_key=ceph_generated_key, - trusted_ip_list=trusted_ips)): - _run_cephadm.assert_called_with( - 'test', - f'iscsi.{iscsi_daemon_id}', - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": f'iscsi.{iscsi_daemon_id}', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [api_port], - }, - "meta": { - 'service_name': f'iscsi.{pool}', - 'ports': [api_port], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "config": "", - "keyring": f"[client.iscsi.{iscsi_daemon_id}]\nkey = None\n", - "files": { - "iscsi-gateway.cfg": iscsi_gateway_conf, - }, - } - }), - error_ok=True, - use_current_daemon_image=False, - ) - - expected_cert_call = call({ - 'prefix': 'config-key set', - 'key': f'iscsi/client.iscsi.{iscsi_daemon_id}/iscsi-gateway.crt', - 'val': ceph_generated_cert, - }) - expected_key_call = call({ - 'prefix': 'config-key set', - 'key': f'iscsi/client.iscsi.{iscsi_daemon_id}/iscsi-gateway.key', - 'val': ceph_generated_key, - }) - - cephadm_module.check_mon_command.assert_has_calls( - [expected_cert_call, expected_key_call], - any_order=True - ) - - -class TestNVMEOFService: - - mgr = FakeMgr() - nvmeof_service = NvmeofService(mgr) - - nvmeof_spec = NvmeofServiceSpec(service_type='nvmeof', service_id="a") - nvmeof_spec.daemon_type = 'nvmeof' - nvmeof_spec.daemon_id = "a" - nvmeof_spec.spec = MagicMock() - nvmeof_spec.spec.daemon_type = 'nvmeof' - - mgr.spec_store = MagicMock() - mgr.spec_store.all_specs.get.return_value = nvmeof_spec - - def test_nvmeof_client_caps(self): - pass - - @patch('cephadm.utils.resolve_ip') - def test_nvmeof_dashboard_config(self, mock_resolve_ip): - pass - - @patch("cephadm.inventory.Inventory.get_addr", lambda _, __: '192.168.100.100') - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator.get_unique_name") - def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrchestrator): - - pool = 'testpool' - group = 'mygroup' - nvmeof_daemon_id = f'{pool}.{group}.test.qwert' - tgt_cmd_extra_args = '--cpumask=0xFF --msg-mempool-size=524288' - default_port = 5500 - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - _get_name.return_value = nvmeof_daemon_id - - nvmeof_gateway_conf = f"""# This file is generated by cephadm. -[gateway] -name = client.nvmeof.{nvmeof_daemon_id} -group = {group} -addr = 192.168.100.100 -port = {default_port} -enable_auth = False -state_update_notify = True -state_update_interval_sec = 5 -break_update_interval_sec = 25 -enable_spdk_discovery_controller = False -encryption_key = /encryption.key -rebalance_period_sec = 7 -max_gws_in_grp = 16 -max_ns_to_change_lb_grp = 8 -enable_prometheus_exporter = True -prometheus_exporter_ssl = False -prometheus_port = 10008 -prometheus_stats_interval = 10 -prometheus_frequency_slow_down_factor = 3.0 -prometheus_cycles_to_adjust_speed = 3 -prometheus_startup_delay = 240 -prometheus_connection_list_cache_expiration = 60 -verify_nqns = True -verify_keys = True -verify_listener_ip = True -# This is a development flag, do not change it -abort_on_errors = True -# This is a development flag, do not change it -abort_on_update_error = True -# This is a development flag, do not change it -omap_file_ignore_unlock_errors = False -# This is a development flag, do not change it -omap_file_lock_on_read = True -omap_file_lock_duration = 40 -omap_file_lock_retries = 30 -omap_file_lock_retry_sleep_interval = 1.0 -omap_file_update_reloads = 10 -omap_file_update_attempts = 500 -allowed_consecutive_spdk_ping_failures = 1 -spdk_ping_interval_in_seconds = 2.0 -ping_spdk_under_lock = False -enable_monitor_client = True -max_hosts_per_namespace = 16 -max_namespaces_with_netmask = 1000 -max_subsystems = 128 -max_hosts = 2048 -max_namespaces = 4096 -max_namespaces_per_subsystem = 512 -max_hosts_per_subsystem = 128 -subsystem_cache_expiration = 30 -force_tls = False -# This is a development flag, do not change it -max_message_length_in_mb = 4 -io_stats_enabled = True - -[gateway-logs] -log_level = INFO -log_files_enabled = True -log_files_rotation_enabled = True -verbose_log_messages = True -max_log_file_size_in_mb = 10 -max_log_files_count = 20 -max_log_directory_backups = 10 -log_directory = /var/log/ceph/ - -[discovery] -addr = 192.168.100.100 -port = 8009 -# This is a development flag, do not change it -abort_on_errors = True -bind_retries_limit = 10 -bind_sleep_interval = 0.5 - -[ceph] -pool = {pool} -config_file = /etc/ceph/ceph.conf -id = nvmeof.{nvmeof_daemon_id} - -[mtls] -server_key = /server.key -client_key = /client.key -server_cert = /server.cert -client_cert = /client.cert -root_ca_cert = /root.ca.cert - -[spdk] -tgt_path = /usr/local/bin/nvmf_tgt -rpc_socket_dir = /var/tmp/ -rpc_socket_name = spdk.sock -timeout = 60.0 -cluster_connections = 32 -protocol_log_level = WARNING -conn_retries = 10 -transports = tcp -transport_tcp_options = {{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}} -enable_dsa_acceleration = False -rbd_with_crc32c = True -tgt_cmd_extra_args = {tgt_cmd_extra_args} -qos_timeslice_in_usecs = 0 -notifications_interval = 60 - -[monitor] -timeout = 1.0\n""" - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, NvmeofServiceSpec(service_id=f'{pool}.{group}', - tgt_cmd_extra_args=tgt_cmd_extra_args, - group=group, - pool=pool)): - _run_cephadm.assert_called_with( - 'test', - f'nvmeof.{nvmeof_daemon_id}', - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": "nvmeof.testpool.mygroup.test.qwert", - "image": "", - "deploy_arguments": [], - "params": { - "tcp_ports": [5500, 4420, 8009, 10008] - }, - "meta": { - "service_name": "nvmeof.testpool.mygroup", - "ports": [5500, 4420, 8009, 10008], - "ip": None, - "deployed_by": [], - "rank": None, - "rank_generation": None, - "extra_container_args": None, - "extra_entrypoint_args": None - }, - "config_blobs": { - "config": "", - "keyring": "[client.nvmeof.testpool.mygroup.test.qwert]\nkey = None\n", - "files": { - "ceph-nvmeof.conf": nvmeof_gateway_conf - } - } - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) - def test_validate_no_group_duplicate_on_apply(self, cephadm_module: CephadmOrchestrator): - nvmeof_spec_group1 = NvmeofServiceSpec( - service_id='testpool.testgroup', - group='testgroup', - pool='testpool' - ) - nvmeof_spec_also_group1 = NvmeofServiceSpec( - service_id='testpool2.testgroup', - group='testgroup', - pool='testpool2' - ) - with with_host(cephadm_module, 'test'): - out = cephadm_module._apply_service_spec(nvmeof_spec_group1) - assert out == 'Scheduled nvmeof.testpool.testgroup update...' - nvmeof_specs = cephadm_module.spec_store.get_by_service_type('nvmeof') - assert len(nvmeof_specs) == 1 - assert nvmeof_specs[0].spec.service_name() == 'nvmeof.testpool.testgroup' - with pytest.raises( - OrchestratorError, - match='Cannot create nvmeof service with group testgroup. That group is already ' - 'being used by the service nvmeof.testpool.testgroup' - ): - cephadm_module._apply_service_spec(nvmeof_spec_also_group1) - assert len(cephadm_module.spec_store.get_by_service_type('nvmeof')) == 1 - - @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) - def test_validate_service_id_matches_group_on_apply(self, cephadm_module: CephadmOrchestrator): - matching_nvmeof_spec_group_service_id = NvmeofServiceSpec( - service_id='pool1.right_group', - group='right_group', - pool='pool1' - ) - mismatch_nvmeof_spec_group_service_id = NvmeofServiceSpec( - service_id='pool2.wrong_group', - group='right_group', - pool='pool2' - ) - matching_nvmeof_spec_group_service_id_with_dot = NvmeofServiceSpec( - service_id='pool3.right.group', - group='right.group', - pool='pool3' - ) - mismatch_nvmeof_spec_group_service_id_with_dot = NvmeofServiceSpec( - service_id='pool4.wrong.group', - group='right.group', - pool='pool4' - ) - with with_host(cephadm_module, 'test'): - cephadm_module._apply_service_spec(matching_nvmeof_spec_group_service_id) - with pytest.raises( - OrchestratorError, - match='The \'nvmeof\' service id/name must end with \'.\'. Found ' - 'group name \'right_group\' and service id \'pool2.wrong_group\'' - ): - cephadm_module._apply_service_spec(mismatch_nvmeof_spec_group_service_id) - cephadm_module._apply_service_spec(matching_nvmeof_spec_group_service_id_with_dot) - with pytest.raises( - OrchestratorError, - match='The \'nvmeof\' service id/name must end with \'.\'. Found ' - 'group name \'right.group\' and service id \'pool4.wrong.group\'' - ): - cephadm_module._apply_service_spec(mismatch_nvmeof_spec_group_service_id_with_dot) - - -class TestMonitoring: - def _get_config(self, url: str) -> str: - - return f""" - # This file is generated by cephadm. - # See https://prometheus.io/docs/alerting/configuration/ for documentation. - - global: - resolve_timeout: 5m - http_config: - tls_config: - insecure_skip_verify: true - - route: - receiver: 'default' - routes: - - group_by: ['alertname'] - group_wait: 10s - group_interval: 10s - repeat_interval: 1h - receiver: 'ceph-dashboard' - - receivers: - - name: 'default' - webhook_configs: - - name: 'custom-receiver' - webhook_configs: - - name: 'ceph-dashboard' - webhook_configs: - - url: '{url}/api/prometheus_receiver' - """ - - @pytest.mark.parametrize( - "dashboard_url,expected_yaml_url", - [ - # loopback address - ("http://[::1]:8080", "http://localhost:8080"), - # IPv6 - ( - "http://[2001:db8:4321:0000:0000:0000:0000:0000]:8080", - "http://[2001:db8:4321:0000:0000:0000:0000:0000]:8080", - ), - # IPv6 to FQDN - ( - "http://[2001:db8:4321:0000:0000:0000:0000:0000]:8080", - "http://mgr.fqdn.test:8080", - ), - # IPv4 - ( - "http://192.168.0.123:8080", - "http://192.168.0.123:8080", - ), - # IPv4 to FQDN - ( - "http://192.168.0.123:8080", - "http://mgr.fqdn.test:8080", - ), - ], - ) - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("mgr_module.MgrModule.get") - @patch("socket.getfqdn") - def test_alertmanager_config( - self, - mock_getfqdn, - mock_get, - _run_cephadm, - cephadm_module: CephadmOrchestrator, - dashboard_url, - expected_yaml_url, - ): - _run_cephadm.side_effect = async_side_effect(("{}", "", 0)) - mock_get.return_value = {"services": {"dashboard": dashboard_url}} - purl = urllib.parse.urlparse(expected_yaml_url) - mock_getfqdn.return_value = purl.hostname - - with with_host(cephadm_module, "test"): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - }, - }) - with with_service(cephadm_module, AlertManagerSpec('alertmanager', - networks=['1.2.3.0/24'], - only_bind_port_on_networks=True)): - y = dedent(self._get_config(expected_yaml_url)).lstrip() - _run_cephadm.assert_called_with( - 'test', - "alertmanager.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'alertmanager.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9093, 9094], - 'port_ips': {"9094": "1.2.3.1"}, - }, - "meta": { - 'service_name': 'alertmanager', - 'ports': [9093, 9094], - 'ip': '1.2.3.1', - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "alertmanager.yml": y, - }, - "peers": [], - "use_url_prefix": False, - "ip_to_bind_to": "1.2.3.1", - } - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("socket.getfqdn") - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch("cephadm.services.monitoring.password_hash", lambda password: 'alertmanager_password_hash') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: 'cephadm_root_cert') - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials('mycert', 'mykey')) - def test_alertmanager_config_when_mgmt_gw_enabled(self, _get_fqdn, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - fqdn = 'host1.test' - _get_fqdn.return_value = fqdn - - with with_host(cephadm_module, 'test'): - cephadm_module.secure_monitoring_stack = True - cephadm_module.set_store(AlertmanagerService.USER_CFG_KEY, 'alertmanager_user') - cephadm_module.set_store(AlertmanagerService.PASS_CFG_KEY, 'alertmanager_plain_password') - - cephadm_module.cache.update_host_networks('test', { - 'fd12:3456:789a::/64': { - 'if0': ['fd12:3456:789a::10'] - }, - }) - with with_service(cephadm_module, MgmtGatewaySpec("mgmt-gateway")) as _, \ - with_service(cephadm_module, AlertManagerSpec('alertmanager', - networks=['fd12:3456:789a::/64'], - only_bind_port_on_networks=True)): - - y = dedent(""" - # This file is generated by cephadm. - # See https://prometheus.io/docs/alerting/configuration/ for documentation. - - global: - resolve_timeout: 5m - http_config: - tls_config: - ca_file: root_cert.pem - cert_file: alertmanager.crt - key_file: alertmanager.key - - route: - receiver: 'default' - routes: - - group_by: ['alertname'] - group_wait: 10s - group_interval: 10s - repeat_interval: 1h - receiver: 'ceph-dashboard' - - receivers: - - name: 'default' - webhook_configs: - - name: 'custom-receiver' - webhook_configs: - - name: 'ceph-dashboard' - webhook_configs: - - url: 'https://host_fqdn:29443/internal/dashboard/api/prometheus_receiver' - """).lstrip() - - web_config = dedent(""" - tls_server_config: - cert_file: alertmanager.crt - key_file: alertmanager.key - client_auth_type: RequireAndVerifyClientCert - client_ca_file: root_cert.pem - basic_auth_users: - alertmanager_user: alertmanager_password_hash - """).lstrip() - - _run_cephadm.assert_called_with( - 'test', - "alertmanager.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'alertmanager.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9093, 9094], - 'port_ips': {"9094": "fd12:3456:789a::10"} - }, - "meta": { - 'service_name': 'alertmanager', - 'ports': [9093, 9094], - 'ip': 'fd12:3456:789a::10', - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "alertmanager.yml": y, - 'alertmanager.crt': 'mycert', - 'alertmanager.key': 'mykey', - 'web.yml': web_config, - 'root_cert.pem': 'cephadm_root_cert' - }, - 'peers': [], - 'web_config': '/etc/alertmanager/web.yml', - "use_url_prefix": True, - "ip_to_bind_to": "fd12:3456:789a::10", - } - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("socket.getfqdn") - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch("cephadm.services.monitoring.password_hash", lambda password: 'alertmanager_password_hash') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: 'cephadm_root_cert') - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials('mycert', 'mykey')) - def test_alertmanager_config_security_enabled(self, _get_fqdn, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - fqdn = 'host1.test' - _get_fqdn.return_value = fqdn - - with with_host(cephadm_module, 'test'): - cephadm_module.secure_monitoring_stack = True - cephadm_module.set_store(AlertmanagerService.USER_CFG_KEY, 'alertmanager_user') - cephadm_module.set_store(AlertmanagerService.PASS_CFG_KEY, 'alertmanager_plain_password') - with with_service(cephadm_module, AlertManagerSpec()): - - y = dedent(f""" - # This file is generated by cephadm. - # See https://prometheus.io/docs/alerting/configuration/ for documentation. - - global: - resolve_timeout: 5m - http_config: - tls_config: - ca_file: root_cert.pem - cert_file: alertmanager.crt - key_file: alertmanager.key - - route: - receiver: 'default' - routes: - - group_by: ['alertname'] - group_wait: 10s - group_interval: 10s - repeat_interval: 1h - receiver: 'ceph-dashboard' - - receivers: - - name: 'default' - webhook_configs: - - name: 'custom-receiver' - webhook_configs: - - name: 'ceph-dashboard' - webhook_configs: - - url: 'http://{fqdn}:8080/api/prometheus_receiver' - """).lstrip() - - web_config = dedent(""" - tls_server_config: - cert_file: alertmanager.crt - key_file: alertmanager.key - basic_auth_users: - alertmanager_user: alertmanager_password_hash - """).lstrip() - - _run_cephadm.assert_called_with( - 'test', - "alertmanager.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'alertmanager.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9093, 9094], - }, - "meta": { - 'service_name': 'alertmanager', - 'ports': [9093, 9094], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "alertmanager.yml": y, - 'alertmanager.crt': 'mycert', - 'alertmanager.key': 'mykey', - 'web.yml': web_config, - 'root_cert.pem': 'cephadm_root_cert' - }, - 'peers': [], - 'web_config': '/etc/alertmanager/web.yml', - "use_url_prefix": False, - "ip_to_bind_to": "", - } - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @pytest.mark.parametrize( - "user_data", - [ - ({'webhook_urls': ['http://foo.com:9999', 'http://bar.com:1111']}), - ({'default_webhook_urls': ['http://bar.com:9999', 'http://foo.com:1111']}), - ({'default_webhook_urls': ['http://bar.com:9999', 'http://foo.com:1111'], - 'webhook_urls': ['http://foo.com:9999', 'http://bar.com:1111']}), - ], - ) - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("socket.getfqdn") - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch("cephadm.services.monitoring.password_hash", lambda password: 'alertmanager_password_hash') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: 'cephadm_root_cert') - @patch('cephadm.cert_mgr.CertMgr.generate_cert', lambda instance, fqdn, ip: ('mycert', 'mykey')) - def test_alertmanager_config_custom_webhook_urls( - self, - _get_fqdn, - _run_cephadm, - cephadm_module: CephadmOrchestrator, - user_data: Dict[str, List[str]] - ): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - cephadm_module.set_store(AlertmanagerService.USER_CFG_KEY, 'alertmanager_user') - cephadm_module.set_store(AlertmanagerService.PASS_CFG_KEY, 'alertmanager_plain_password') - fqdn = 'host1.test' - _get_fqdn.return_value = fqdn - - print(user_data) - - urls = [] - if 'default_webhook_urls' in user_data: - urls += user_data['default_webhook_urls'] - if 'webhook_urls' in user_data: - urls += user_data['webhook_urls'] - tab_over = ' ' * 18 # since we'll be inserting this into an indented string - webhook_configs_str = '\n'.join(f'{tab_over}- url: \'{u}\'' for u in urls) - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, AlertManagerSpec(user_data=user_data)): - - y = dedent(f""" - # This file is generated by cephadm. - # See https://prometheus.io/docs/alerting/configuration/ for documentation. - - global: - resolve_timeout: 5m - http_config: - tls_config: - insecure_skip_verify: true - - route: - receiver: 'default' - routes: - - group_by: ['alertname'] - group_wait: 10s - group_interval: 10s - repeat_interval: 1h - receiver: 'ceph-dashboard' - continue: true - - group_by: ['alertname'] - group_wait: 10s - group_interval: 10s - repeat_interval: 1h - receiver: 'custom-receiver' - - receivers: - - name: 'default' - webhook_configs: - - name: 'custom-receiver' - webhook_configs: -{webhook_configs_str} - - name: 'ceph-dashboard' - webhook_configs: - - url: 'http://{fqdn}:8080/api/prometheus_receiver' - """).lstrip() - - _run_cephadm.assert_called_with( - 'test', - "alertmanager.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'alertmanager.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9093, 9094], - }, - "meta": { - 'service_name': 'alertmanager', - 'ports': [9093, 9094], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "alertmanager.yml": y, - }, - 'peers': [], - "use_url_prefix": False, - "ip_to_bind_to": "", - } - }), - use_current_daemon_image=False, - error_ok=True, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("socket.getfqdn") - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: 'cephadm_root_cert') - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials('mycert', 'mykey')) - @patch('cephadm.services.cephadmservice.CephExporterService.get_keyring_with_caps', Mock(return_value='[client.ceph-exporter.test]\nkey = fake-secret\n')) - def test_ceph_exporter_config_security_enabled(self, _get_fqdn, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - fqdn = 'host1.test' - _get_fqdn.return_value = fqdn - - with with_host(cephadm_module, 'test'): - cephadm_module.secure_monitoring_stack = True - with with_service(cephadm_module, CephExporterSpec()): - _run_cephadm.assert_called_with('test', 'ceph-exporter.test', - ['_orch', 'deploy'], [], - stdin=json.dumps({ - "fsid": "fsid", - "name": "ceph-exporter.test", - "image": "", - "deploy_arguments": [], - "params": {"tcp_ports": [9926]}, - "meta": { - "service_name": "ceph-exporter", - "ports": [9926], - "ip": None, - "deployed_by": [], - "rank": None, - "rank_generation": None, - "extra_container_args": None, - "extra_entrypoint_args": None - }, - "config_blobs": { - "config": "", - "keyring": "[client.ceph-exporter.test]\nkey = fake-secret\n", - "prio-limit": "5", - "stats-period": "5", - "https_enabled": True, - "files": { - "ceph-exporter.crt": "mycert", - "ceph-exporter.key": "mykey"}}}), - error_ok=True, - use_current_daemon_image=False) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("mgr_module.MgrModule.get") - @patch("socket.getfqdn") - def test_node_exporter_config_without_mgmt_gw( - self, - mock_getfqdn, - mock_get, - _run_cephadm, - cephadm_module: CephadmOrchestrator, - ): - _run_cephadm.side_effect = async_side_effect(("{}", "", 0)) - fqdn = 'host1.test' - mock_getfqdn.return_value = fqdn - - with with_host(cephadm_module, "test"): - with with_service(cephadm_module, MonitoringSpec('node-exporter')): - _run_cephadm.assert_called_with( - 'test', - "node-exporter.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'node-exporter.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9100], - }, - "meta": { - 'service_name': 'node-exporter', - 'ports': [9100], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": {} - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("socket.getfqdn") - def test_node_exporter_config_with_mgmt_gw( - self, - mock_getfqdn, - _run_cephadm, - cephadm_module: CephadmOrchestrator, - ): - _run_cephadm.side_effect = async_side_effect(("{}", "", 0)) - mock_getfqdn.return_value = 'host1.test' - - y = dedent(""" - tls_server_config: - cert_file: node_exporter.crt - key_file: node_exporter.key - client_auth_type: RequireAndVerifyClientCert - client_ca_file: root_cert.pem - """).lstrip() - - with with_host(cephadm_module, "test"): - with with_service(cephadm_module, MgmtGatewaySpec("mgmt-gateway")) as _, \ - with_service(cephadm_module, MonitoringSpec('node-exporter')): - _run_cephadm.assert_called_with( - 'test', - "node-exporter.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'node-exporter.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9100], - }, - "meta": { - 'service_name': 'node-exporter', - 'ports': [9100], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "web.yml": y, - 'root_cert.pem': f"{cephadm_root_ca}", - 'node_exporter.crt': f"{ceph_generated_cert}", - 'node_exporter.key': f"{ceph_generated_key}", - }, - 'web_config': '/etc/node-exporter/web.yml', - } - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator._get_mgr_ips", lambda _: ['192.168.100.100', '::1']) - def test_prometheus_config_security_disabled(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - pool = 'testpool' - group = 'mygroup' - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), rgw_frontend_type='beast') - with with_host(cephadm_module, 'test'): - # host "test" needs to have networks for keepalive to be placed - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - }, - }) - with with_service(cephadm_module, MonitoringSpec('node-exporter')) as _, \ - with_service(cephadm_module, CephExporterSpec('ceph-exporter')) as _, \ - with_service(cephadm_module, s) as _, \ - with_service(cephadm_module, AlertManagerSpec('alertmanager')) as _, \ - with_service(cephadm_module, IngressSpec(service_id='ingress', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_ip="1.2.3.4/32", - backend_service='rgw.foo', - enable_stats=True)) as _, \ - with_service(cephadm_module, NvmeofServiceSpec(service_id=f'{pool}.{group}', - group=group, - pool=pool)) as _, \ - with_service(cephadm_module, PrometheusSpec('prometheus', - networks=['1.2.3.0/24'], - only_bind_port_on_networks=True)) as _: - - y = dedent(""" - # This file is generated by cephadm. - global: - scrape_interval: 10s - evaluation_interval: 10s - external_labels: - cluster: fsid - - rule_files: - - /etc/prometheus/alerting/* - - alerting: - alertmanagers: - - scheme: http - http_sd_configs: - - url: http://192.168.100.100:8765/sd/prometheus/sd-config?service=alertmanager - - url: http://[::1]:8765/sd/prometheus/sd-config?service=alertmanager - - scrape_configs: - - job_name: 'ceph' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - - source_labels: [instance] - target_label: instance - replacement: 'ceph_cluster' - honor_labels: true - http_sd_configs: - - url: http://192.168.100.100:8765/sd/prometheus/sd-config?service=ceph - - url: http://[::1]:8765/sd/prometheus/sd-config?service=ceph - - - job_name: 'ceph-exporter' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - honor_labels: true - http_sd_configs: - - url: http://192.168.100.100:8765/sd/prometheus/sd-config?service=ceph-exporter - - url: http://[::1]:8765/sd/prometheus/sd-config?service=ceph-exporter - - - job_name: 'ingress' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - honor_labels: true - http_sd_configs: - - url: http://192.168.100.100:8765/sd/prometheus/sd-config?service=ingress - - url: http://[::1]:8765/sd/prometheus/sd-config?service=ingress - - - job_name: 'node-exporter' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - honor_labels: true - http_sd_configs: - - url: http://192.168.100.100:8765/sd/prometheus/sd-config?service=node-exporter - - url: http://[::1]:8765/sd/prometheus/sd-config?service=node-exporter - - - job_name: 'nvmeof' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - honor_labels: true - http_sd_configs: - - url: http://192.168.100.100:8765/sd/prometheus/sd-config?service=nvmeof - - url: http://[::1]:8765/sd/prometheus/sd-config?service=nvmeof - - - """).lstrip() - - _run_cephadm.assert_called_with( - 'test', - "prometheus.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'prometheus.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9095], - 'port_ips': {'8765': '1.2.3.1'} - }, - "meta": { - 'service_name': 'prometheus', - 'ports': [9095], - 'ip': '1.2.3.1', - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "prometheus.yml": y, - "/etc/prometheus/alerting/custom_alerts.yml": "", - }, - 'retention_time': '15d', - 'retention_size': '0', - 'ip_to_bind_to': '1.2.3.1', - "use_url_prefix": False - }, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.module.CephadmOrchestrator.get_unique_name") - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator._get_mgr_ips", lambda _: ['::1']) - @patch("cephadm.services.monitoring.password_hash", lambda password: 'prometheus_password_hash') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: 'cephadm_root_cert') - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials('mycert', 'mykey')) - def test_prometheus_config_security_enabled(self, _run_cephadm, _get_uname, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - _get_uname.return_value = 'test' - pool = 'testpool' - group = 'mygroup' - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), rgw_frontend_type='beast') - smb_spec = SMBSpec(cluster_id='foxtrot', config_uri='rados://.smb/foxtrot/config.json',) - - with with_host(cephadm_module, 'test'): - cephadm_module.secure_monitoring_stack = True - cephadm_module.set_store(PrometheusService.USER_CFG_KEY, 'prometheus_user') - cephadm_module.set_store(PrometheusService.PASS_CFG_KEY, 'prometheus_plain_password') - cephadm_module.set_store(AlertmanagerService.USER_CFG_KEY, 'alertmanager_user') - cephadm_module.set_store(AlertmanagerService.PASS_CFG_KEY, 'alertmanager_plain_password') - cephadm_module.http_server.service_discovery.username = 'sd_user' - cephadm_module.http_server.service_discovery.password = 'sd_password' - # host "test" needs to have networks for keepalive to be placed - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - }, - }) - with with_service(cephadm_module, MonitoringSpec('node-exporter')) as _, \ - with_service(cephadm_module, smb_spec) as _, \ - with_service(cephadm_module, CephExporterSpec('ceph-exporter')) as _, \ - with_service(cephadm_module, s) as _, \ - with_service(cephadm_module, AlertManagerSpec('alertmanager')) as _, \ - with_service(cephadm_module, IngressSpec(service_id='ingress', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_ip="1.2.3.4/32", - backend_service='rgw.foo', - enable_stats=True)) as _, \ - with_service(cephadm_module, NvmeofServiceSpec(service_id=f'{pool}.{group}', - group=group, - pool=pool)) as _, \ - with_service(cephadm_module, PrometheusSpec('prometheus')) as _: - - web_config = dedent(""" - tls_server_config: - cert_file: prometheus.crt - key_file: prometheus.key - basic_auth_users: - prometheus_user: prometheus_password_hash - """).lstrip() - - y = dedent(""" - # This file is generated by cephadm. - global: - scrape_interval: 10s - evaluation_interval: 10s - external_labels: - cluster: fsid - - rule_files: - - /etc/prometheus/alerting/* - - alerting: - alertmanagers: - - scheme: https - basic_auth: - username: alertmanager_user - password: alertmanager_plain_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - path_prefix: '/' - http_sd_configs: - - url: https://[::1]:8765/sd/prometheus/sd-config?service=alertmanager - basic_auth: - username: sd_user - password: sd_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - - scrape_configs: - - job_name: 'ceph' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - - source_labels: [instance] - target_label: instance - replacement: 'ceph_cluster' - scheme: https - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - honor_labels: true - http_sd_configs: - - url: https://[::1]:8765/sd/prometheus/sd-config?service=ceph - basic_auth: - username: sd_user - password: sd_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - - - job_name: 'ceph-exporter' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - scheme: https - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - honor_labels: true - http_sd_configs: - - url: https://[::1]:8765/sd/prometheus/sd-config?service=ceph-exporter - basic_auth: - username: sd_user - password: sd_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - - - job_name: 'ingress' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - scheme: https - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - honor_labels: true - http_sd_configs: - - url: https://[::1]:8765/sd/prometheus/sd-config?service=ingress - basic_auth: - username: sd_user - password: sd_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - - - job_name: 'node-exporter' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - scheme: https - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - honor_labels: true - http_sd_configs: - - url: https://[::1]:8765/sd/prometheus/sd-config?service=node-exporter - basic_auth: - username: sd_user - password: sd_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - - - job_name: 'nvmeof' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - scheme: https - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - honor_labels: true - http_sd_configs: - - url: https://[::1]:8765/sd/prometheus/sd-config?service=nvmeof - basic_auth: - username: sd_user - password: sd_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - - - job_name: 'smb' - relabel_configs: - - source_labels: [__address__] - target_label: cluster - replacement: fsid - scheme: https - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - honor_labels: true - http_sd_configs: - - url: https://[::1]:8765/sd/prometheus/sd-config?service=smb - basic_auth: - username: sd_user - password: sd_password - tls_config: - ca_file: root_cert.pem - cert_file: prometheus.crt - key_file: prometheus.key - - - """).lstrip() - - _run_cephadm.assert_called_with( - 'test', - "prometheus.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'prometheus.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9095], - }, - "meta": { - 'service_name': 'prometheus', - 'ports': [9095], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - 'files': { - 'prometheus.yml': y, - 'root_cert.pem': 'cephadm_root_cert', - 'web.yml': web_config, - 'prometheus.crt': 'mycert', - 'prometheus.key': 'mykey', - "/etc/prometheus/alerting/custom_alerts.yml": "", - }, - 'retention_time': '15d', - 'retention_size': '0', - 'ip_to_bind_to': '', - "use_url_prefix": False, - 'web_config': '/etc/prometheus/web.yml' - }, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_loki_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, MonitoringSpec('loki')) as _: - - y = dedent(""" - # This file is generated by cephadm. - auth_enabled: false - - server: - http_listen_port: 3100 - grpc_listen_port: 8080 - - common: - path_prefix: /loki - storage: - filesystem: - chunks_directory: /loki/chunks - rules_directory: /loki/rules - replication_factor: 1 - ring: - instance_addr: 127.0.0.1 - kvstore: - store: inmemory - - schema_config: - configs: - - from: 2020-10-24 - store: boltdb-shipper - object_store: filesystem - schema: v11 - index: - prefix: index_ - period: 24h - - from: 2024-05-03 - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h""").lstrip() - - _run_cephadm.assert_called_with( - 'test', - "loki.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'loki.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [3100], - }, - "meta": { - 'service_name': 'loki', - 'ports': [3100], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "loki.yml": y - }, - }, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_promtail_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, ServiceSpec('mgr')) as _, \ - with_service(cephadm_module, MonitoringSpec('promtail')) as _: - - y = dedent(""" - # This file is generated by cephadm. - server: - http_listen_port: 9080 - grpc_listen_port: 0 - - positions: - filename: /tmp/positions.yaml - - clients: - - url: http://:3100/loki/api/v1/push - - scrape_configs: - - job_name: system - static_configs: - - labels: - job: Cluster Logs - __path__: /var/log/ceph/**/*.log""").lstrip() - - _run_cephadm.assert_called_with( - 'test', - "promtail.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'promtail.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9080], - }, - "meta": { - 'service_name': 'promtail', - 'ports': [9080], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": { - "promtail.yml": y - }, - }, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '1::4') - @patch("cephadm.module.CephadmOrchestrator.get_fqdn", lambda a, b: 'host_fqdn') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - def test_grafana_config_with_mgmt_gw_and_ouath2_proxy(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(("{}", "", 0)) - - def inline_certificate(multi_line_cert): - """ - Converts a multi-line certificate into a one-line string with escaped newlines. - """ - return '\\n'.join([line.strip() for line in multi_line_cert.splitlines()]) - - oneline_cephadm_root_ca = inline_certificate(cephadm_root_ca) - oneline_ceph_generated_cert = inline_certificate(ceph_generated_cert) - oneline_ceph_generated_key = inline_certificate(ceph_generated_key) - - y = dedent(f""" - # This file is generated by cephadm. - apiVersion: 1 - - deleteDatasources: - - name: 'Dashboard1' - orgId: 1 - - datasources: - - name: 'Dashboard1' - type: 'prometheus' - access: 'proxy' - orgId: 1 - url: 'https://host_fqdn:29443/internal/prometheus' - basicAuth: true - isDefault: true - editable: false - basicAuthUser: admin - jsonData: - graphiteVersion: "1.1" - tlsAuth: false - tlsAuthWithCACert: true - tlsSkipVerify: false - secureJsonData: - basicAuthPassword: admin - tlsCACert: "{oneline_cephadm_root_ca}" - tlsClientCert: "{oneline_ceph_generated_cert}" - tlsClientKey: "{oneline_ceph_generated_key}" - - - name: 'Loki' - type: 'loki' - access: 'proxy' - url: '' - basicAuth: false - isDefault: false - editable: false""").lstrip() - - oauth2_spec = OAuth2ProxySpec(provider_display_name='my_idp_provider', - client_id='my_client_id', - client_secret='my_client_secret', - oidc_issuer_url='http://192.168.10.10:8888/dex', - cookie_secret='kbAEM9opAmuHskQvt0AW8oeJRaOM2BYy5Loba0kZ0SQ=', - ssl_cert=ceph_generated_cert, - ssl_key=ceph_generated_key) - - with with_host(cephadm_module, "test"): - cephadm_module.cert_mgr.save_cert('grafana_ssl_cert', ceph_generated_cert, host='test') - cephadm_module.cert_mgr.save_key('grafana_ssl_key', ceph_generated_key, host='test') - with with_service(cephadm_module, PrometheusSpec("prometheus")) as _, \ - with_service(cephadm_module, MgmtGatewaySpec("mgmt-gateway")) as _, \ - with_service(cephadm_module, oauth2_spec) as _, \ - with_service(cephadm_module, ServiceSpec("mgr")) as _, with_service( - cephadm_module, GrafanaSpec("grafana") - ) as _: - files = { - 'grafana.ini': dedent(""" - # This file is generated by cephadm. - [users] - default_theme = light - [auth.anonymous] - enabled = true - org_name = 'Main Org.' - org_role = 'Viewer' - [server] - domain = 'host_fqdn' - protocol = https - cert_file = /etc/grafana/certs/cert_file - cert_key = /etc/grafana/certs/cert_key - http_port = 3000 - http_addr = - root_url = %(protocol)s://%(domain)s:%(http_port)s/grafana/ - serve_from_sub_path = true - [snapshots] - external_enabled = false - [security] - disable_initial_admin_creation = true - cookie_secure = true - cookie_samesite = none - allow_embedding = true - [auth] - disable_login_form = true - [auth.proxy] - enabled = true - header_name = X-WEBAUTH-USER - header_property = username - auto_sign_up = true - sync_ttl = 15 - whitelist = 1::4 - headers_encoded = false - enable_login_token = false - headers = Role:X-WEBAUTH-ROLE - [analytics] - check_for_updates = false - reporting_enabled = false - [plugins] - check_for_plugin_updates = false - public_key_retrieval_disabled = true""").lstrip(), # noqa: W291 - "provisioning/datasources/ceph-dashboard.yml": y, - 'certs/cert_file': dedent(f""" - # generated by cephadm\n{ceph_generated_cert}""").lstrip(), - 'certs/cert_key': dedent(f""" - # generated by cephadm\n{ceph_generated_key}""").lstrip(), - 'provisioning/dashboards/default.yml': dedent(""" - # This file is generated by cephadm. - apiVersion: 1 - - providers: - - name: 'Ceph Dashboard' - orgId: 1 - folder: '' - type: file - disableDeletion: false - updateIntervalSeconds: 3 - editable: false - options: - path: '/etc/grafana/provisioning/dashboards'""").lstrip(), - } - - _run_cephadm.assert_called_with( - 'test', - "grafana.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'grafana.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [3000], - }, - "meta": { - 'service_name': 'grafana', - 'ports': [3000], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": files, - }, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '1::4') - @patch("cephadm.module.CephadmOrchestrator.get_fqdn", lambda a, b: 'host_fqdn') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - def test_grafana_config_with_mgmt_gw(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(("{}", "", 0)) - - def inline_certificate(multi_line_cert): - """ - Converts a multi-line certificate into a one-line string with escaped newlines. - """ - return '\\n'.join([line.strip() for line in multi_line_cert.splitlines()]) - - oneline_cephadm_root_ca = inline_certificate(cephadm_root_ca) - oneline_ceph_generated_cert = inline_certificate(ceph_generated_cert) - oneline_ceph_generated_key = inline_certificate(ceph_generated_key) - - y = dedent(f""" - # This file is generated by cephadm. - apiVersion: 1 - - deleteDatasources: - - name: 'Dashboard1' - orgId: 1 - - datasources: - - name: 'Dashboard1' - type: 'prometheus' - access: 'proxy' - orgId: 1 - url: 'https://host_fqdn:29443/internal/prometheus' - basicAuth: true - isDefault: true - editable: false - basicAuthUser: admin - jsonData: - graphiteVersion: "1.1" - tlsAuth: false - tlsAuthWithCACert: true - tlsSkipVerify: false - secureJsonData: - basicAuthPassword: admin - tlsCACert: "{oneline_cephadm_root_ca}" - tlsClientCert: "{oneline_ceph_generated_cert}" - tlsClientKey: "{oneline_ceph_generated_key}" - - - name: 'Loki' - type: 'loki' - access: 'proxy' - url: '' - basicAuth: false - isDefault: false - editable: false""").lstrip() - - with with_host(cephadm_module, "test"): - with with_service( - cephadm_module, PrometheusSpec("prometheus") - ) as _, with_service(cephadm_module, MgmtGatewaySpec("mgmt-gateway")) as _, \ - with_service(cephadm_module, ServiceSpec("mgr")) as _, with_service( - cephadm_module, GrafanaSpec("grafana") - ) as _: - cephadm_module.cert_mgr.save_self_signed_cert_key_pair('grafana', - TLSCredentials(ceph_generated_cert, ceph_generated_key), - host='test') - files = { - 'grafana.ini': dedent(""" - # This file is generated by cephadm. - [users] - default_theme = light - [auth.anonymous] - enabled = true - org_name = 'Main Org.' - org_role = 'Viewer' - [server] - domain = 'host_fqdn' - protocol = https - cert_file = /etc/grafana/certs/cert_file - cert_key = /etc/grafana/certs/cert_key - http_port = 3000 - http_addr = - root_url = %(protocol)s://%(domain)s:%(http_port)s/grafana/ - serve_from_sub_path = true - [snapshots] - external_enabled = false - [security] - disable_initial_admin_creation = true - cookie_secure = true - cookie_samesite = none - allow_embedding = true - [analytics] - check_for_updates = false - reporting_enabled = false - [plugins] - check_for_plugin_updates = false - public_key_retrieval_disabled = true""").lstrip(), # noqa: W291 - "provisioning/datasources/ceph-dashboard.yml": y, - 'certs/cert_file': dedent(f""" - # generated by cephadm\n{ceph_generated_cert}""").lstrip(), - 'certs/cert_key': dedent(f""" - # generated by cephadm\n{ceph_generated_key}""").lstrip(), - 'provisioning/dashboards/default.yml': dedent(""" - # This file is generated by cephadm. - apiVersion: 1 - - providers: - - name: 'Ceph Dashboard' - orgId: 1 - folder: '' - type: file - disableDeletion: false - updateIntervalSeconds: 3 - editable: false - options: - path: '/etc/grafana/provisioning/dashboards'""").lstrip(), - } - - _run_cephadm.assert_called_with( - 'test', - "grafana.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'grafana.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [3000], - }, - "meta": { - 'service_name': 'grafana', - 'ports': [3000], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": files, - }, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '1::4') - @patch("cephadm.module.CephadmOrchestrator.get_fqdn", lambda a, b: 'host_fqdn') - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - def test_grafana_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(("{}", "", 0)) - - with with_host(cephadm_module, "test"): - with with_service( - cephadm_module, PrometheusSpec("prometheus") - ) as _, with_service(cephadm_module, ServiceSpec("mgr")) as _, with_service( - cephadm_module, GrafanaSpec("grafana") - ) as _: - cephadm_module.cert_mgr.save_self_signed_cert_key_pair('grafana', - TLSCredentials(ceph_generated_cert, ceph_generated_key), - host='test') - files = { - 'grafana.ini': dedent(""" - # This file is generated by cephadm. - [users] - default_theme = light - [auth.anonymous] - enabled = true - org_name = 'Main Org.' - org_role = 'Viewer' - [server] - domain = 'host_fqdn' - protocol = https - cert_file = /etc/grafana/certs/cert_file - cert_key = /etc/grafana/certs/cert_key - http_port = 3000 - http_addr = - [snapshots] - external_enabled = false - [security] - disable_initial_admin_creation = true - cookie_secure = true - cookie_samesite = none - allow_embedding = true - [analytics] - check_for_updates = false - reporting_enabled = false - [plugins] - check_for_plugin_updates = false - public_key_retrieval_disabled = true""").lstrip(), # noqa: W291 - 'provisioning/datasources/ceph-dashboard.yml': dedent(""" - # This file is generated by cephadm. - apiVersion: 1 - - deleteDatasources: - - name: 'Dashboard1' - orgId: 1 - - datasources: - - name: 'Dashboard1' - type: 'prometheus' - access: 'proxy' - orgId: 1 - url: 'http://host_fqdn:9095' - basicAuth: false - isDefault: true - editable: false - - - name: 'Loki' - type: 'loki' - access: 'proxy' - url: '' - basicAuth: false - isDefault: false - editable: false""").lstrip(), - 'certs/cert_file': dedent(f""" - # generated by cephadm\n{ceph_generated_cert}""").lstrip(), - 'certs/cert_key': dedent(f""" - # generated by cephadm\n{ceph_generated_key}""").lstrip(), - 'provisioning/dashboards/default.yml': dedent(""" - # This file is generated by cephadm. - apiVersion: 1 - - providers: - - name: 'Ceph Dashboard' - orgId: 1 - folder: '' - type: file - disableDeletion: false - updateIntervalSeconds: 3 - editable: false - options: - path: '/etc/grafana/provisioning/dashboards'""").lstrip(), - } - - _run_cephadm.assert_called_with( - 'test', - "grafana.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'grafana.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [3000], - }, - "meta": { - 'service_name': 'grafana', - 'ports': [3000], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "files": files, - }, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) - def test_grafana_initial_admin_pw(self, cephadm_module: CephadmOrchestrator): - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, ServiceSpec('mgr')) as _, \ - with_service(cephadm_module, GrafanaSpec(initial_admin_password='secure')): - out = service_registry.get_service('grafana').generate_config( - CephadmDaemonDeploySpec('test', 'daemon', 'grafana')) - assert out == ( - { - 'files': - { - 'grafana.ini': - '# This file is generated by cephadm.\n' - '[users]\n' - ' default_theme = light\n' - '[auth.anonymous]\n' - ' enabled = true\n' - " org_name = 'Main Org.'\n" - " org_role = 'Viewer'\n" - '[server]\n' - " domain = 'host_fqdn'\n" - ' protocol = https\n' - ' cert_file = /etc/grafana/certs/cert_file\n' - ' cert_key = /etc/grafana/certs/cert_key\n' - ' http_port = 3000\n' - ' http_addr = \n' - '[snapshots]\n' - ' external_enabled = false\n' - '[security]\n' - ' admin_user = admin\n' - ' admin_password = secure\n' - ' cookie_secure = true\n' - ' cookie_samesite = none\n' - ' allow_embedding = true\n' - '[analytics]\n' - ' check_for_updates = false\n' - ' reporting_enabled = false\n' - '[plugins]\n' - ' check_for_plugin_updates = false\n' - ' public_key_retrieval_disabled = true', - 'provisioning/datasources/ceph-dashboard.yml': - "# This file is generated by cephadm.\n" - "apiVersion: 1\n\n" - 'deleteDatasources:\n\n' - 'datasources:\n\n' - " - name: 'Loki'\n" - " type: 'loki'\n" - " access: 'proxy'\n" - " url: ''\n" - ' basicAuth: false\n' - ' isDefault: false\n' - ' editable: false', - 'certs/cert_file': ANY, - 'certs/cert_key': ANY, - 'provisioning/dashboards/default.yml': - '# This file is generated by cephadm.\n' - 'apiVersion: 1\n\n' - 'providers:\n' - " - name: 'Ceph Dashboard'\n" - ' orgId: 1\n' - " folder: ''\n" - ' type: file\n' - ' disableDeletion: false\n' - ' updateIntervalSeconds: 3\n' - ' editable: false\n' - ' options:\n' - " path: '/etc/grafana/provisioning/dashboards'" - }}, ['secure_monitoring_stack:False']) - - @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) - def test_grafana_no_anon_access(self, cephadm_module: CephadmOrchestrator): - # with anonymous_access set to False, expecting the [auth.anonymous] section - # to not be present in the grafana config. Note that we require an initial_admin_password - # to be provided when anonymous_access is False - cephadm_module._init_cert_mgr() - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, ServiceSpec('mgr')) as _, \ - with_service(cephadm_module, GrafanaSpec(anonymous_access=False, initial_admin_password='secure')): - out = service_registry.get_service('grafana').generate_config( - CephadmDaemonDeploySpec('test', 'daemon', 'grafana')) - assert out == ( - { - 'files': - { - 'grafana.ini': - '# This file is generated by cephadm.\n' - '[users]\n' - ' default_theme = light\n' - '[server]\n' - " domain = 'host_fqdn'\n" - ' protocol = https\n' - ' cert_file = /etc/grafana/certs/cert_file\n' - ' cert_key = /etc/grafana/certs/cert_key\n' - ' http_port = 3000\n' - ' http_addr = \n' - '[snapshots]\n' - ' external_enabled = false\n' - '[security]\n' - ' admin_user = admin\n' - ' admin_password = secure\n' - ' cookie_secure = true\n' - ' cookie_samesite = none\n' - ' allow_embedding = true\n' - '[analytics]\n' - ' check_for_updates = false\n' - ' reporting_enabled = false\n' - '[plugins]\n' - ' check_for_plugin_updates = false\n' - ' public_key_retrieval_disabled = true', - 'provisioning/datasources/ceph-dashboard.yml': - "# This file is generated by cephadm.\n" - "apiVersion: 1\n\n" - 'deleteDatasources:\n\n' - 'datasources:\n\n' - " - name: 'Loki'\n" - " type: 'loki'\n" - " access: 'proxy'\n" - " url: ''\n" - ' basicAuth: false\n' - ' isDefault: false\n' - ' editable: false', - 'certs/cert_file': ANY, - 'certs/cert_key': ANY, - 'provisioning/dashboards/default.yml': - '# This file is generated by cephadm.\n' - 'apiVersion: 1\n\n' - 'providers:\n' - " - name: 'Ceph Dashboard'\n" - ' orgId: 1\n' - " folder: ''\n" - ' type: file\n' - ' disableDeletion: false\n' - ' updateIntervalSeconds: 3\n' - ' editable: false\n' - ' options:\n' - " path: '/etc/grafana/provisioning/dashboards'" - }}, ['secure_monitoring_stack:False']) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_monitoring_ports(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test'): - - yaml_str = """service_type: alertmanager -service_name: alertmanager -placement: - count: 1 -spec: - port: 4200 -""" - yaml_file = yaml.safe_load(yaml_str) - spec = ServiceSpec.from_json(yaml_file) - - with patch("cephadm.services.monitoring.AlertmanagerService.generate_config", return_value=({}, [])): - with with_service(cephadm_module, spec): - - CephadmServe(cephadm_module)._check_daemons() - - _run_cephadm.assert_called_with( - 'test', - "alertmanager.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'alertmanager.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [4200, 9094], - }, - "meta": { - 'service_name': 'alertmanager', - 'ports': [4200, 9094], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": {}, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - -class TestRGWService: - - @pytest.mark.parametrize( - "frontend, ssl, extra_args, expected", - [ - ('beast', False, ['tcp_nodelay=1'], - 'beast endpoint=[fd00:fd00:fd00:3000::1]:80 tcp_nodelay=1'), - ('beast', True, ['tcp_nodelay=0', 'max_header_size=65536'], - 'beast ssl_endpoint=[fd00:fd00:fd00:3000::1]:443 ssl_certificate=config://rgw/cert/rgw.foo tcp_nodelay=0 max_header_size=65536'), - ('civetweb', False, [], 'civetweb port=[fd00:fd00:fd00:3000::1]:80'), - ('civetweb', True, None, - 'civetweb port=[fd00:fd00:fd00:3000::1]:443s ssl_certificate=config://rgw/cert/rgw.foo'), - ] - ) - @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) - def test_rgw_update(self, frontend, ssl, extra_args, expected, cephadm_module: CephadmOrchestrator): - with with_host(cephadm_module, 'host1'): - cephadm_module.cache.update_host_networks('host1', { - 'fd00:fd00:fd00:3000::/64': { - 'if0': ['fd00:fd00:fd00:3000::1'] - } - }) - s = RGWSpec(service_id="foo", - networks=['fd00:fd00:fd00:3000::/64'], - ssl=ssl, - rgw_frontend_type=frontend, - rgw_frontend_extra_args=extra_args) - with with_service(cephadm_module, s) as dds: - _, f, _ = cephadm_module.check_mon_command({ - 'prefix': 'config get', - 'who': f'client.{dds[0]}', - 'key': 'rgw_frontends', - }) - assert f == expected - - @pytest.mark.parametrize( - "disable_sync_traffic", - [ - (True), - (False), - ] - ) - @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) - def test_rgw_disable_sync_traffic(self, disable_sync_traffic, cephadm_module: CephadmOrchestrator): - with with_host(cephadm_module, 'host1'): - s = RGWSpec(service_id="foo", - disable_multisite_sync_traffic=disable_sync_traffic) - with with_service(cephadm_module, s) as dds: - _, f, _ = cephadm_module.check_mon_command({ - 'prefix': 'config get', - 'who': f'client.{dds[0]}', - 'key': 'rgw_run_sync_thread', - }) - assert f == ('false' if disable_sync_traffic else 'true') - - -class TestMonService: - - def test_set_crush_locations(self, cephadm_module: CephadmOrchestrator): - mgr = FakeMgr() - mon_service = MonService(mgr) - mon_spec = ServiceSpec(service_type='mon', crush_locations={'vm-00': ['datacenter=a', 'rack=1'], 'vm-01': ['datacenter=a'], 'vm-02': ['datacenter=b', 'rack=3']}) - - mon_daemons = [ - DaemonDescription(daemon_type='mon', daemon_id='vm-00', hostname='vm-00'), - DaemonDescription(daemon_type='mon', daemon_id='vm-01', hostname='vm-01'), - DaemonDescription(daemon_type='mon', daemon_id='vm-02', hostname='vm-02') - ] - mon_service.set_crush_locations(mon_daemons, mon_spec) - assert 'vm-00' in mgr.set_mon_crush_locations - assert mgr.set_mon_crush_locations['vm-00'] == ['datacenter=a', 'rack=1'] - assert 'vm-01' in mgr.set_mon_crush_locations - assert mgr.set_mon_crush_locations['vm-01'] == ['datacenter=a'] - assert 'vm-02' in mgr.set_mon_crush_locations - assert mgr.set_mon_crush_locations['vm-02'] == ['datacenter=b', 'rack=3'] - - -class TestSNMPGateway: - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_snmp_v2c_deployment(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - spec = SNMPGatewaySpec( - snmp_version='V2c', - snmp_destination='192.168.1.1:162', - credentials={ - 'snmp_community': 'public' - }) - - config = { - "destination": spec.snmp_destination, - "snmp_version": spec.snmp_version, - "snmp_community": spec.credentials.get('snmp_community') - } - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'test', - "snmp-gateway.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'snmp-gateway.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9464], - }, - "meta": { - 'service_name': 'snmp-gateway', - 'ports': [9464], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_snmp_v2c_with_port(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - spec = SNMPGatewaySpec( - snmp_version='V2c', - snmp_destination='192.168.1.1:162', - credentials={ - 'snmp_community': 'public' - }, - port=9465) - - config = { - "destination": spec.snmp_destination, - "snmp_version": spec.snmp_version, - "snmp_community": spec.credentials.get('snmp_community') - } - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'test', - "snmp-gateway.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'snmp-gateway.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9465], - }, - "meta": { - 'service_name': 'snmp-gateway', - 'ports': [9465], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_snmp_v3nopriv_deployment(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - spec = SNMPGatewaySpec( - snmp_version='V3', - snmp_destination='192.168.1.1:162', - engine_id='8000C53F00000000', - credentials={ - 'snmp_v3_auth_username': 'myuser', - 'snmp_v3_auth_password': 'mypassword' - }) - - config = { - 'destination': spec.snmp_destination, - 'snmp_version': spec.snmp_version, - 'snmp_v3_auth_protocol': 'SHA', - 'snmp_v3_auth_username': 'myuser', - 'snmp_v3_auth_password': 'mypassword', - 'snmp_v3_engine_id': '8000C53F00000000' - } - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'test', - "snmp-gateway.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'snmp-gateway.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9464], - }, - "meta": { - 'service_name': 'snmp-gateway', - 'ports': [9464], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_snmp_v3priv_deployment(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - spec = SNMPGatewaySpec( - snmp_version='V3', - snmp_destination='192.168.1.1:162', - engine_id='8000C53F00000000', - auth_protocol='MD5', - privacy_protocol='AES', - credentials={ - 'snmp_v3_auth_username': 'myuser', - 'snmp_v3_auth_password': 'mypassword', - 'snmp_v3_priv_password': 'mysecret', - }) - - config = { - 'destination': spec.snmp_destination, - 'snmp_version': spec.snmp_version, - 'snmp_v3_auth_protocol': 'MD5', - 'snmp_v3_auth_username': spec.credentials.get('snmp_v3_auth_username'), - 'snmp_v3_auth_password': spec.credentials.get('snmp_v3_auth_password'), - 'snmp_v3_engine_id': '8000C53F00000000', - 'snmp_v3_priv_protocol': spec.privacy_protocol, - 'snmp_v3_priv_password': spec.credentials.get('snmp_v3_priv_password'), - } - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'test', - "snmp-gateway.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'snmp-gateway.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9464], - }, - "meta": { - 'service_name': 'snmp-gateway', - 'ports': [9464], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - -class TestIngressService: - - @pytest.mark.parametrize( - "enable_haproxy_protocol", - [False, True], - ) - @patch("cephadm.inventory.Inventory.get_addr") - @patch("cephadm.utils.resolve_ip") - @patch("cephadm.inventory.HostCache.get_daemons_by_service") - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_config_nfs_multiple_nfs_same_rank( - self, - _run_cephadm, - _get_daemons_by_service, - _resolve_ip, _get_addr, - cephadm_module: CephadmOrchestrator, - enable_haproxy_protocol: bool, - ): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - def fake_resolve_ip(hostname: str) -> str: - if hostname == 'host1': - return '192.168.122.111' - elif hostname == 'host2': - return '192.168.122.222' - else: - return 'xxx.xxx.xxx.xxx' - _resolve_ip.side_effect = fake_resolve_ip - - def fake_get_addr(hostname: str) -> str: - return hostname - _get_addr.side_effect = fake_get_addr - - nfs_service = NFSServiceSpec( - service_id="foo", - placement=PlacementSpec( - count=1, - hosts=['host1', 'host2']), - port=12049, - enable_haproxy_protocol=enable_haproxy_protocol, - ) - - ispec = IngressSpec( - service_type='ingress', - service_id='nfs.foo', - backend_service='nfs.foo', - frontend_port=2049, - monitor_port=9049, - virtual_ip='192.168.122.100/24', - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - enable_haproxy_protocol=enable_haproxy_protocol, - enable_stats=True, - placement=PlacementSpec( - hosts=['host1']) - ) - - cephadm_module.spec_store._specs = { - 'nfs.foo': nfs_service, - 'ingress.nfs.foo': ispec - } - cephadm_module.spec_store.spec_created = { - 'nfs.foo': datetime_now(), - 'ingress.nfs.foo': datetime_now() - } - - # in both test cases we'll do here, we want only the ip - # for the host1 nfs daemon as we'll end up giving that - # one higher rank_generation but the same rank as the one - # on host2 - haproxy_txt = ( - '# This file is generated by cephadm.\n' - 'global\n' - ' log 127.0.0.1 local2\n' - ' chroot /var/lib/haproxy\n' - ' pidfile /var/lib/haproxy/haproxy.pid\n' - ' maxconn 8000\n' - ' daemon\n' - ' stats socket /var/lib/haproxy/stats\n\n' - 'defaults\n' - ' mode tcp\n' - ' log global\n' - ' timeout queue 1m\n' - ' timeout connect 10s\n' - ' timeout client 1m\n' - ' timeout server 1m\n' - ' timeout check 10s\n' - ' maxconn 8000\n\n' - 'frontend stats\n' - ' mode http\n' - ' bind 192.168.122.100:9049\n' - ' bind host1:9049\n' - ' stats enable\n' - ' stats uri /stats\n' - ' stats refresh 10s\n' - ' stats auth admin:12345\n' - ' http-request use-service prometheus-exporter if { path /metrics }\n' - ' monitor-uri /health\n\n' - 'frontend frontend\n' - ' bind 192.168.122.100:2049\n' - ' option tcplog\n' - ' default_backend backend\n\n' - 'peers haproxy_peers\n' - ' peer host1 host1:1024\n\n' - 'backend backend\n' - ' mode tcp\n' - ' balance roundrobin\n' - ' stick-table type ip size 200k expire 30m peers haproxy_peers\n' - ' stick on src\n' - ' hash-type consistent\n' - ) - if enable_haproxy_protocol: - haproxy_txt += ' default-server send-proxy-v2\n' - haproxy_txt += ' server nfs.foo.0 192.168.122.111:12049 check\n' - haproxy_expected_conf = { - 'files': {'haproxy.cfg': haproxy_txt} - } - - # verify we get the same cfg regardless of the order in which the nfs daemons are returned - # in this case both nfs are rank 0, so it should only take the one with rank_generation 1 a.k.a - # the one on host1 - nfs_daemons = [ - DaemonDescription(daemon_type='nfs', daemon_id='foo.0.1.host1.qwerty', hostname='host1', rank=0, rank_generation=1, ports=[12049]), - DaemonDescription(daemon_type='nfs', daemon_id='foo.0.0.host2.abcdef', hostname='host2', rank=0, rank_generation=0, ports=[12049]) - ] - _get_daemons_by_service.return_value = nfs_daemons - - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='host1', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_generated_conf = haproxy_generated_conf[0] - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf['files']['haproxy.cfg'].splitlines()] - exp_config_line = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - - assert gen_config_lines == exp_config_line - - # swapping order now, should still pick out the one with the higher rank_generation - # in this case both nfs are rank 0, so it should only take the one with rank_generation 1 a.k.a - # the one on host1 - nfs_daemons = [ - DaemonDescription(daemon_type='nfs', daemon_id='foo.0.0.host2.abcdef', hostname='host2', rank=0, rank_generation=0, ports=[12049]), - DaemonDescription(daemon_type='nfs', daemon_id='foo.0.1.host1.qwerty', hostname='host1', rank=0, rank_generation=1, ports=[12049]) - ] - _get_daemons_by_service.return_value = nfs_daemons - - haproxy_generated_conf, _ = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='host1', daemon_id='ingress', service_name=ispec.service_name())) - - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - - assert gen_config_lines == exp_config_lines - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': [ - '1.2.3.4', # simulate already assigned VIP - '1.2.3.1', # simulate interface IP - ] - } - }) - - # the ingress backend - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32", - enable_stats=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the keepalived conf based on the specified spec - keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - keepalived_expected_conf = { - 'files': - { - 'keepalived.conf': - '# This file is generated by cephadm.\n' - 'global_defs {\n ' - 'enable_script_security\n ' - 'script_user root\n' - '}\n\n' - 'vrrp_script check_backend {\n ' - 'script "/usr/bin/curl http://1.2.3.7:8999/health"\n ' - 'weight -20\n ' - 'interval 2\n ' - 'rise 2\n ' - 'fall 2\n}\n\n' - 'vrrp_instance VI_0 {\n ' - 'state MASTER\n ' - 'priority 100\n ' - 'interface if0\n ' - 'virtual_router_id 50\n ' - 'advert_int 1\n ' - 'authentication {\n ' - 'auth_type PASS\n ' - 'auth_pass 12345\n ' - '}\n ' - 'unicast_src_ip 1.2.3.1\n ' - 'unicast_peer {\n ' - '}\n ' - 'virtual_ipaddress {\n ' - '1.2.3.4/32 dev if0\n ' - '}\n ' - 'track_script {\n ' - 'check_backend\n }\n' - '}\n' - } - } - - # check keepalived config - assert keepalived_generated_conf[0] == keepalived_expected_conf - - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode http\n ' - 'log global\n ' - 'option httplog\n ' - 'option dontlognull\n ' - 'option http-server-close\n ' - 'option forwardfor except 127.0.0.0/8\n ' - 'option redispatch\n ' - 'retries 3\n ' - 'timeout queue 20s\n ' - 'timeout connect 5s\n ' - 'timeout http-request 1s\n ' - 'timeout http-keep-alive 5s\n ' - 'timeout client 30s\n ' - 'timeout server 30s\n ' - 'timeout check 5s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n ' - 'mode http\n ' - 'bind 1.2.3.4:8999\n ' - 'bind 1.2.3.7:8999\n ' - 'stats enable\n ' - 'stats uri /stats\n ' - 'stats refresh 10s\n ' - 'stats auth admin:12345\n ' - 'http-request use-service prometheus-exporter if { path /metrics }\n ' - 'monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089\n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'option forwardfor\n ' - 'balance static-rr\n ' - 'option httpchk HEAD / HTTP/1.0\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' - } - } - - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - - assert gen_config_lines == exp_config_lines - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_config_ssl_rgw(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - with with_host(cephadm_module, 'test'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - } - }) - - # the ingress backend - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast', rgw_frontend_port=443, ssl=True) - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32", - enable_stats=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the keepalived conf based on the specified spec - keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - keepalived_expected_conf = { - 'files': - { - 'keepalived.conf': - '# This file is generated by cephadm.\n' - 'global_defs {\n ' - 'enable_script_security\n ' - 'script_user root\n' - '}\n\n' - 'vrrp_script check_backend {\n ' - 'script "/usr/bin/curl http://[1::4]:8999/health"\n ' - 'weight -20\n ' - 'interval 2\n ' - 'rise 2\n ' - 'fall 2\n}\n\n' - 'vrrp_instance VI_0 {\n ' - 'state MASTER\n ' - 'priority 100\n ' - 'interface if0\n ' - 'virtual_router_id 50\n ' - 'advert_int 1\n ' - 'authentication {\n ' - 'auth_type PASS\n ' - 'auth_pass 12345\n ' - '}\n ' - 'unicast_src_ip 1.2.3.1\n ' - 'unicast_peer {\n ' - '}\n ' - 'virtual_ipaddress {\n ' - '1.2.3.4/32 dev if0\n ' - '}\n ' - 'track_script {\n ' - 'check_backend\n }\n' - '}\n' - } - } - - # check keepalived config - assert keepalived_generated_conf[0] == keepalived_expected_conf - - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode http\n ' - 'log global\n ' - 'option httplog\n ' - 'option dontlognull\n ' - 'option http-server-close\n ' - 'option forwardfor except 127.0.0.0/8\n ' - 'option redispatch\n ' - 'retries 3\n ' - 'timeout queue 20s\n ' - 'timeout connect 5s\n ' - 'timeout http-request 1s\n ' - 'timeout http-keep-alive 5s\n ' - 'timeout client 30s\n ' - 'timeout server 30s\n ' - 'timeout check 5s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n ' - 'mode http\n ' - 'bind 1.2.3.4:8999\n ' - 'bind 1::4:8999\n ' - 'stats enable\n ' - 'stats uri /stats\n ' - 'stats refresh 10s\n ' - 'stats auth admin:12345\n ' - 'http-request use-service prometheus-exporter if { path /metrics }\n ' - 'monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089\n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'option forwardfor\n ' - 'default-server ssl\n ' - 'default-server verify none\n ' - 'balance static-rr\n ' - 'option httpchk HEAD / HTTP/1.0\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100 inter 2s\n' - } - } - - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - assert gen_config_lines == exp_config_lines - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_haproxy_config_rgw_tcp_mode(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - with with_host(cephadm_module, 'test'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - } - }) - - # the ingress backend - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast', rgw_frontend_port=443, ssl=True) - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32", - use_tcp_mode_over_rgw=True, - enable_stats=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode tcp\n ' - 'log global\n ' - 'timeout queue 1m\n ' - 'timeout connect 10s\n ' - 'timeout client 1m\n ' - 'timeout server 1m\n ' - 'timeout check 10s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n ' - 'mode http\n ' - 'bind 1.2.3.4:8999\n ' - 'bind 1::4:8999\n ' - 'stats enable\n ' - 'stats uri /stats\n ' - 'stats refresh 10s\n ' - 'stats auth admin:12345\n ' - 'http-request use-service prometheus-exporter if { path /metrics }\n ' - 'monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089 \n ' - 'option tcplog\n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'mode tcp\n ' - 'balance roundrobin\n ' - 'hash-type consistent\n ' - 'option ssl-hello-chk\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100 inter 2s\n' - } - } - - assert haproxy_generated_conf[0] == haproxy_expected_conf - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - } - }) - - # Check the ingress with multiple VIPs - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32", - enable_stats=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the keepalived conf based on the specified spec - # Test with only 1 IP on the list, as it will fail with more VIPS but only one host. - keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - keepalived_expected_conf = { - 'files': - { - 'keepalived.conf': - '# This file is generated by cephadm.\n' - 'global_defs {\n ' - 'enable_script_security\n ' - 'script_user root\n' - '}\n\n' - 'vrrp_script check_backend {\n ' - 'script "/usr/bin/curl http://1.2.3.7:8999/health"\n ' - 'weight -20\n ' - 'interval 2\n ' - 'rise 2\n ' - 'fall 2\n}\n\n' - 'vrrp_instance VI_0 {\n ' - 'state MASTER\n ' - 'priority 100\n ' - 'interface if0\n ' - 'virtual_router_id 50\n ' - 'advert_int 1\n ' - 'authentication {\n ' - 'auth_type PASS\n ' - 'auth_pass 12345\n ' - '}\n ' - 'unicast_src_ip 1.2.3.1\n ' - 'unicast_peer {\n ' - '}\n ' - 'virtual_ipaddress {\n ' - '1.2.3.4/32 dev if0\n ' - '}\n ' - 'track_script {\n ' - 'check_backend\n }\n' - '}\n' - } - } - - # check keepalived config - assert keepalived_generated_conf[0] == keepalived_expected_conf - - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode http\n ' - 'log global\n ' - 'option httplog\n ' - 'option dontlognull\n ' - 'option http-server-close\n ' - 'option forwardfor except 127.0.0.0/8\n ' - 'option redispatch\n ' - 'retries 3\n ' - 'timeout queue 20s\n ' - 'timeout connect 5s\n ' - 'timeout http-request 1s\n ' - 'timeout http-keep-alive 5s\n ' - 'timeout client 30s\n ' - 'timeout server 30s\n ' - 'timeout check 5s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n ' - 'mode http\n ' - 'bind 1.2.3.4:8999\n ' - 'bind 1.2.3.7:8999\n ' - 'stats enable\n ' - 'stats uri /stats\n ' - 'stats refresh 10s\n ' - 'stats auth admin:12345\n ' - 'http-request use-service prometheus-exporter if { path /metrics }\n ' - 'monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089 \n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'option forwardfor\n ' - 'balance static-rr\n ' - 'option httpchk HEAD / HTTP/1.0\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' - } - } - - assert haproxy_generated_conf[0] == haproxy_expected_conf - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_keepalive_config_multi_interface_vips(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - with with_host(cephadm_module, 'test', addr='1.2.3.1'): - with with_host(cephadm_module, 'test2', addr='1.2.3.2'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - }, - '100.100.100.0/24': { - 'if1': ['100.100.100.1'] - } - }) - cephadm_module.cache.update_host_networks('test2', { - '1.2.3.0/24': { - 'if0': ['1.2.3.2'] - }, - '100.100.100.0/24': { - 'if1': ['100.100.100.2'] - } - }) - - # Check the ingress with multiple VIPs - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - placement=PlacementSpec(hosts=['test', 'test2']), - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_ips_list=["1.2.3.100/24", "100.100.100.100/24"], - enable_stats=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - keepalived_expected_conf = { - 'files': - { - 'keepalived.conf': - '# This file is generated by cephadm.\n' - 'global_defs {\n ' - 'enable_script_security\n ' - 'script_user root\n' - '}\n\n' - 'vrrp_script check_backend {\n ' - 'script "/usr/bin/curl http://1.2.3.1:8999/health"\n ' - 'weight -20\n ' - 'interval 2\n ' - 'rise 2\n ' - 'fall 2\n}\n\n' - 'vrrp_instance VI_0 {\n ' - 'state MASTER\n ' - 'priority 100\n ' - 'interface if0\n ' - 'virtual_router_id 50\n ' - 'advert_int 1\n ' - 'authentication {\n ' - 'auth_type PASS\n ' - 'auth_pass 12345\n ' - '}\n ' - 'unicast_src_ip 1.2.3.1\n ' - 'unicast_peer {\n ' - '1.2.3.2\n ' - '}\n ' - 'virtual_ipaddress {\n ' - '1.2.3.100/24 dev if0\n ' - '}\n ' - 'track_script {\n ' - 'check_backend\n }\n' - '}\n' - 'vrrp_instance VI_1 {\n ' - 'state BACKUP\n ' - 'priority 90\n ' - 'interface if1\n ' - 'virtual_router_id 51\n ' - 'advert_int 1\n ' - 'authentication {\n ' - 'auth_type PASS\n ' - 'auth_pass 12345\n ' - '}\n ' - 'unicast_src_ip 100.100.100.1\n ' - 'unicast_peer {\n ' - '100.100.100.2\n ' - '}\n ' - 'virtual_ipaddress {\n ' - '100.100.100.100/24 dev if1\n ' - '}\n ' - 'track_script {\n ' - 'check_backend\n }\n' - '}\n' - } - } - - # check keepalived config - assert keepalived_generated_conf[0] == keepalived_expected_conf - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_keepalive_interface_host_filtering(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - # we need to make sure keepalive daemons will have an interface - # on the hosts we deploy them on in order to set up their VIP. - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.1'): - with with_host(cephadm_module, 'test2', addr='1.2.3.2'): - with with_host(cephadm_module, 'test3', addr='1.2.3.3'): - with with_host(cephadm_module, 'test4', addr='1.2.3.3'): - # setup "test" and "test4" to have all the necessary interfaces, - # "test2" to have one of them (should still be filtered) - # and "test3" to have none of them - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - }, - '100.100.100.0/24': { - 'if1': ['100.100.100.1'] - } - }) - cephadm_module.cache.update_host_networks('test2', { - '1.2.3.0/24': { - 'if0': ['1.2.3.2'] - }, - }) - cephadm_module.cache.update_host_networks('test4', { - '1.2.3.0/24': { - 'if0': ['1.2.3.4'] - }, - '100.100.100.0/24': { - 'if1': ['100.100.100.4'] - } - }) - - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - placement=PlacementSpec(hosts=['test', 'test2', 'test3', 'test4']), - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_ips_list=["1.2.3.100/24", "100.100.100.100/24"], - enable_stats=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # since we're never actually going to refresh the host here, - # check the tmp daemons to see what was placed during the apply - daemons = cephadm_module.cache._get_tmp_daemons() - keepalive_daemons = [d for d in daemons if d.daemon_type == 'keepalived'] - hosts_deployed_on = [d.hostname for d in keepalive_daemons] - assert 'test' in hosts_deployed_on - assert 'test2' not in hosts_deployed_on - assert 'test3' not in hosts_deployed_on - assert 'test4' in hosts_deployed_on - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_haproxy_port_ips(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.4/32'] - } - }) - - # Check the ingress with multiple VIPs - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ip = '1.2.3.100' - frontend_port = 8089 - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=frontend_port, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_ip=f"{ip}/24", - enable_stats=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the haproxy conf based on the specified spec - haproxy_daemon_spec = service_registry.get_service('ingress').prepare_create( - CephadmDaemonDeploySpec( - host='test', - daemon_type='haproxy', - daemon_id='ingress', - service_name=ispec.service_name())) - - assert haproxy_daemon_spec.port_ips == {str(frontend_port): ip} - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.nfs.NFSService.fence_old_ranks", MagicMock()) - @patch("cephadm.services.nfs.NFSService.run_grace_tool", MagicMock()) - @patch("cephadm.services.nfs.NFSService.purge", MagicMock()) - @patch("cephadm.services.nfs.NFSService.create_rados_config_obj", MagicMock()) - def test_keepalive_only_nfs_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - } - }) - - # Check the ingress with multiple VIPs - s = NFSServiceSpec(service_id="foo", placement=PlacementSpec(count=1), - virtual_ip='1.2.3.0/24') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='nfs.foo', - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_ip='1.2.3.0/24', - keepalive_only=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - nfs_generated_conf, _ = service_registry.get_service('nfs').generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='foo.test.0.0', service_name=s.service_name())) - ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] - assert "Bind_addr = 1.2.3.0/24" in ganesha_conf - - keepalived_generated_conf = service_registry.get_service('ingress').keepalived_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - keepalived_expected_conf = { - 'files': - { - 'keepalived.conf': - '# This file is generated by cephadm.\n' - 'global_defs {\n ' - 'enable_script_security\n ' - 'script_user root\n' - '}\n\n' - 'vrrp_script check_backend {\n ' - 'script "/usr/bin/false"\n ' - 'weight -20\n ' - 'interval 2\n ' - 'rise 2\n ' - 'fall 2\n}\n\n' - 'vrrp_instance VI_0 {\n ' - 'state MASTER\n ' - 'priority 100\n ' - 'interface if0\n ' - 'virtual_router_id 50\n ' - 'advert_int 1\n ' - 'authentication {\n ' - 'auth_type PASS\n ' - 'auth_pass 12345\n ' - '}\n ' - 'unicast_src_ip 1.2.3.1\n ' - 'unicast_peer {\n ' - '}\n ' - 'virtual_ipaddress {\n ' - '1.2.3.0/24 dev if0\n ' - '}\n ' - 'track_script {\n ' - 'check_backend\n }\n' - '}\n' - } - } - - # check keepalived config - assert keepalived_generated_conf[0] == keepalived_expected_conf - - @patch("cephadm.services.nfs.NFSService.fence_old_ranks", MagicMock()) - @patch("cephadm.services.nfs.NFSService.run_grace_tool", MagicMock()) - @patch("cephadm.services.nfs.NFSService.purge", MagicMock()) - @patch("cephadm.services.nfs.NFSService.create_rados_config_obj", MagicMock()) - @patch("cephadm.inventory.Inventory.keys") - @patch("cephadm.inventory.Inventory.get_addr") - @patch("cephadm.utils.resolve_ip") - @patch("cephadm.inventory.HostCache.get_daemons_by_service") - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_config_nfs_proxy_protocol( - self, - _run_cephadm, - _get_daemons_by_service, - _resolve_ip, - _get_addr, - _inventory_keys, - cephadm_module: CephadmOrchestrator, - ): - """Verify that setting enable_haproxy_protocol for both ingress and - nfs services sets the desired configuration parameters in both - the haproxy config and nfs ganesha config. - """ - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - def fake_resolve_ip(hostname: str) -> str: - if hostname in ('host1', "192.168.122.111"): - return '192.168.122.111' - elif hostname in ('host2', '192.168.122.222'): - return '192.168.122.222' - else: - raise KeyError(hostname) - _resolve_ip.side_effect = fake_resolve_ip - _get_addr.side_effect = fake_resolve_ip - - def fake_keys(): - return ['host1', 'host2'] - _inventory_keys.side_effect = fake_keys - - nfs_service = NFSServiceSpec( - service_id="foo", - placement=PlacementSpec( - count=1, - hosts=['host1', 'host2']), - port=12049, - enable_haproxy_protocol=True, - enable_nlm=True, - ) - - ispec = IngressSpec( - service_type='ingress', - service_id='nfs.foo', - backend_service='nfs.foo', - frontend_port=2049, - monitor_port=9049, - virtual_ip='192.168.122.100/24', - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - enable_haproxy_protocol=True, - enable_stats=True, - placement=PlacementSpec( - count=1, - hosts=['host1', 'host2']), - ) - - cephadm_module.spec_store._specs = { - 'nfs.foo': nfs_service, - 'ingress.nfs.foo': ispec - } - cephadm_module.spec_store.spec_created = { - 'nfs.foo': datetime_now(), - 'ingress.nfs.foo': datetime_now() - } - - haproxy_txt = ( - '# This file is generated by cephadm.\n' - 'global\n' - ' log 127.0.0.1 local2\n' - ' chroot /var/lib/haproxy\n' - ' pidfile /var/lib/haproxy/haproxy.pid\n' - ' maxconn 8000\n' - ' daemon\n' - ' stats socket /var/lib/haproxy/stats\n\n' - 'defaults\n' - ' mode tcp\n' - ' log global\n' - ' timeout queue 1m\n' - ' timeout connect 10s\n' - ' timeout client 1m\n' - ' timeout server 1m\n' - ' timeout check 10s\n' - ' maxconn 8000\n\n' - 'frontend stats\n' - ' mode http\n' - ' bind 192.168.122.100:9049\n' - ' bind 192.168.122.111:9049\n' - ' stats enable\n' - ' stats uri /stats\n' - ' stats refresh 10s\n' - ' stats auth admin:12345\n' - ' http-request use-service prometheus-exporter if { path /metrics }\n' - ' monitor-uri /health\n\n' - 'frontend frontend\n' - ' bind 192.168.122.100:2049\n' - ' option tcplog\n' - ' default_backend backend\n\n' - 'peers haproxy_peers\n' - ' peer host1 192.168.122.111:1024\n' - ' peer host2 192.168.122.222:1024\n\n' - 'backend backend\n' - ' mode tcp\n' - ' balance roundrobin\n' - ' stick-table type ip size 200k expire 30m peers haproxy_peers\n' - ' stick on src\n' - ' hash-type consistent\n' - ' default-server send-proxy-v2\n' - ' server nfs.foo.0 192.168.122.111:12049 check\n' - ) - haproxy_expected_conf = { - 'files': {'haproxy.cfg': haproxy_txt} - } - - nfs_ganesha_txt = ( - "# This file is generated by cephadm.\n" - 'NFS_CORE_PARAM {\n' - ' Enable_NLM = true;\n' - ' Enable_RQUOTA = false;\n' - ' Protocols = 3, 4;\n' - ' mount_path_pseudo = true;\n' - ' Enable_UDP = false;\n' - ' NFS_Port = 2049;\n' - ' allow_set_io_flusher_fail = true;\n' - ' HAProxy_Hosts = 192.168.122.111, 10.10.2.20, 192.168.122.222;\n' - ' Monitoring_Port = 9587;\n' - '}\n' - '\n' - 'NFSv4 {\n' - ' Delegations = false;\n' - ' RecoveryBackend = "rados_cluster";\n' - ' Minor_Versions = 1, 2;\n' - f' Server_Scope = "{cephadm_module._cluster_fsid}-foo";\n' - ' IdmapConf = "/etc/ganesha/idmap.conf";\n' - '}\n' - '\n' - 'RADOS_KV {\n' - ' UserId = "nfs.foo.test.0.0";\n' - ' nodeid = 0;\n' - ' pool = ".nfs";\n' - ' namespace = "foo";\n' - '}\n' - '\n' - 'RADOS_URLS {\n' - ' UserId = "nfs.foo.test.0.0";\n' - ' watch_url = ' - '"rados://.nfs/foo/conf-nfs.foo";\n' - '}\n' - '\n' - 'RGW {\n' - ' cluster = "ceph";\n' - ' name = "client.nfs.foo.test.0.0-rgw";\n' - '}\n' - '\n' - "%url rados://.nfs/foo/conf-nfs.foo" - ) - nfs_expected_conf = { - 'files': {'ganesha.conf': nfs_ganesha_txt, 'idmap.conf': ''}, - 'config': '', - 'extra_args': ['-N', 'NIV_EVENT'], - 'keyring': ( - '[client.nfs.foo.test.0.0]\n' - 'key = None\n' - ), - 'namespace': 'foo', - 'pool': '.nfs', - 'rgw': { - 'cluster': 'ceph', - 'keyring': ( - '[client.nfs.foo.test.0.0-rgw]\n' - 'key = None\n' - ), - 'user': 'nfs.foo.test.0.0-rgw', - }, - 'userid': 'nfs.foo.test.0.0', - } - - nfs_daemons = [ - DaemonDescription( - daemon_type='nfs', - daemon_id='foo.0.1.host1.qwerty', - hostname='host1', - rank=0, - rank_generation=1, - ports=[12049], - ), - DaemonDescription( - daemon_type='nfs', - daemon_id='foo.0.0.host2.abcdef', - hostname='host2', - rank=0, - rank_generation=0, - ports=[12049], - ), - ] - _get_daemons_by_service.return_value = nfs_daemons - - ingress_svc = service_registry.get_service('ingress') - nfs_svc = service_registry.get_service('nfs') - - # add host network info to one host to test the behavior of - # adding all known-good addresses of the host to the list. - cephadm_module.cache.update_host_networks('host1', { - # this one is additional - '10.10.2.0/24': { - 'eth1': ['10.10.2.20'] - }, - # this is redundant and will be skipped - '192.168.122.0/24': { - 'eth0': ['192.168.122.111'] - }, - # this is a link-local address and will be ignored - "fe80::/64": { - "veth0": [ - "fe80::8cf5:25ff:fe1c:d963" - ], - "eth0": [ - "fe80::c7b:cbff:fef6:7370" - ], - "eth1": [ - "fe80::7201:25a7:390b:d9a7" - ] - }, - }) - - haproxy_generated_conf, _ = ingress_svc.haproxy_generate_config( - CephadmDaemonDeploySpec( - host='host1', - daemon_id='ingress', - service_name=ispec.service_name(), - ), - ) - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - assert gen_config_lines == exp_config_lines - - nfs_generated_conf, _ = nfs_svc.generate_config( - CephadmDaemonDeploySpec( - host='test', - daemon_id='foo.test.0.0', - service_name=nfs_service.service_name(), - rank=0, - ), - ) - assert nfs_generated_conf == nfs_expected_conf - - @patch("cephadm.services.nfs.NFSService.fence_old_ranks", MagicMock()) - @patch("cephadm.services.nfs.NFSService.run_grace_tool", MagicMock()) - @patch("cephadm.services.nfs.NFSService.purge", MagicMock()) - @patch("cephadm.services.nfs.NFSService.create_rados_config_obj", MagicMock()) - @patch("cephadm.inventory.Inventory.keys") - @patch("cephadm.inventory.Inventory.get_addr") - @patch("cephadm.utils.resolve_ip") - @patch("cephadm.inventory.HostCache.get_daemons_by_service") - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_haproxy_protocol_nfs_config_with_ip_addrs( - self, - _run_cephadm, - _get_daemons_by_service, - _resolve_ip, - _get_addr, - _inventory_keys, - cephadm_module: CephadmOrchestrator, - ): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - nfs_service = NFSServiceSpec( - service_id="foo", - placement=PlacementSpec( - count=1, - hosts=['host1', 'host2']), - port=12049, - ip_addrs={ - 'host1': '10.10.2.20', - 'host2': '10.10.2.21' - }, - enable_haproxy_protocol=True, - ) - - ispec = IngressSpec( - service_type='ingress', - service_id='nfs.foo', - backend_service='nfs.foo', - frontend_port=2049, - monitor_port=9049, - virtual_ip='192.168.122.100/24', - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - enable_haproxy_protocol=True, - placement=PlacementSpec( - count=1, - hosts=['host1', 'host2']), - - ) - cephadm_module.spec_store._specs = { - 'nfs.foo': nfs_service, - 'ingress.nfs.foo': ispec - } - cephadm_module.spec_store.spec_created = { - 'nfs.foo': datetime_now(), - 'ingress.nfs.foo': datetime_now() - } - nfs_daemons = [ - DaemonDescription( - daemon_type='nfs', - daemon_id='foo.0.1.host1.qwerty', - hostname='host1', - ip='10.10.2.20', - rank=0, - rank_generation=1, - ports=[12049], - ), - DaemonDescription( - daemon_type='nfs', - daemon_id='foo.0.0.host2.abcdef', - hostname='host2', - ip='10.10.2.21', - rank=0, - rank_generation=0, - ports=[12049], - ), - ] - _get_daemons_by_service.return_value = nfs_daemons - - ingress_svc = service_registry.get_service('ingress') - nfs_svc = service_registry.get_service('nfs') - - cephadm_module.cache.update_host_networks('host1', { - # this one is additional - '10.10.2.0/24': { - 'eth1': ['10.10.2.20'] - }, - # this is redundant and will be skipped - '192.168.122.0/24': { - 'eth0': ['192.168.122.111'] - }, - }) - cephadm_module.cache.update_host_networks('host2', { - # this one is additional - '10.10.2.0/24': { - 'eth1': ['10.10.2.22'] - }, - # this is redundant and will be skipped - '192.168.122.0/24': { - 'eth0': ['192.168.122.112'] - }, - }) - - haproxy_generated_conf, _ = ingress_svc.haproxy_generate_config( - CephadmDaemonDeploySpec( - host='host1', - daemon_id='ingress', - service_name=ispec.service_name(), - ), - ) - gen_config_lines = haproxy_generated_conf['files']['haproxy.cfg'] - assert 'server nfs.foo.0 10.10.2.20:12049 check' in gen_config_lines - - nfs_generated_conf, _ = nfs_svc.generate_config( - CephadmDaemonDeploySpec( - host='test', - daemon_id='foo.test.0.0', - service_name=nfs_service.service_name(), - rank=0, - ip='10.10.2.20' - ), - ) - ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] - assert "Bind_addr = 10.10.2.20" in ganesha_conf - - -class TestNFS: - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.nfs.NFSService.fence_old_ranks", MagicMock()) - @patch("cephadm.services.nfs.NFSService.run_grace_tool", MagicMock()) - @patch("cephadm.services.nfs.NFSService.purge", MagicMock()) - @patch("cephadm.services.nfs.NFSService.create_rados_config_obj", MagicMock()) - def test_nfs_config_monitoring_ip(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - } - }) - - nfs_spec = NFSServiceSpec(service_id="foo", placement=PlacementSpec(hosts=['test']), - monitoring_ip_addrs={'test': '1.2.3.1'}) - with with_service(cephadm_module, nfs_spec) as _: - nfs_generated_conf, _ = service_registry.get_service('nfs').generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='foo.test.0.0', service_name=nfs_spec.service_name())) - ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] - assert "Monitoring_Addr = 1.2.3.1" in ganesha_conf - - nfs_spec = NFSServiceSpec(service_id="foo", placement=PlacementSpec(hosts=['test']), - monitoring_networks=['1.2.3.0/24']) - with with_service(cephadm_module, nfs_spec) as _: - nfs_generated_conf, _ = service_registry.get_service('nfs').generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='foo.test.0.0', service_name=nfs_spec.service_name())) - ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] - assert "Monitoring_Addr = 1.2.3.1" in ganesha_conf - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.nfs.NFSService.fence_old_ranks", MagicMock()) - @patch("cephadm.services.nfs.NFSService.run_grace_tool", MagicMock()) - @patch("cephadm.services.nfs.NFSService.purge", MagicMock()) - @patch("cephadm.services.nfs.NFSService.create_rados_config_obj", MagicMock()) - def test_nfs_config_bind_addr(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'host1', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('host1', { - '1.2.3.0/24': { - 'if0': ['1.2.3.7'] - } - }) - - nfs_spec = NFSServiceSpec(service_id="foo", placement=PlacementSpec(hosts=['host1']), - ip_addrs={'host1': '1.2.3.7'}) - with with_service(cephadm_module, nfs_spec, status_running=True) as _: - dds = wait(cephadm_module, cephadm_module.list_daemons()) - daemon_spec = CephadmDaemonDeploySpec.from_daemon_description(dds[0]) - nfs_generated_conf, _ = service_registry.get_service('nfs').generate_config(daemon_spec) - ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] - assert "Bind_addr = 1.2.3.7" in ganesha_conf - - with with_host(cephadm_module, 'host1', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('host1', { - '1.2.3.0/24': { - 'if0': ['1.2.3.7'] - } - }) - nfs_spec = NFSServiceSpec(service_id="foo", placement=PlacementSpec(hosts=['host1']), - networks=['1.2.3.0/24']) - with with_service(cephadm_module, nfs_spec, status_running=True) as _: - dds = wait(cephadm_module, cephadm_module.list_daemons()) - daemon_spec = CephadmDaemonDeploySpec.from_daemon_description(dds[0]) - nfs_generated_conf, _ = service_registry.get_service('nfs').generate_config(daemon_spec) - ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] - assert "Bind_addr = 1.2.3.7" in ganesha_conf - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_without_haproxy_stats(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': [ - '1.2.3.4', # simulate already assigned VIP - '1.2.3.1', # simulate interface IP - ] - } - }) - - # the ingress backend - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32") - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode http\n ' - 'log global\n ' - 'option httplog\n ' - 'option dontlognull\n ' - 'option http-server-close\n ' - 'option forwardfor except 127.0.0.0/8\n ' - 'option redispatch\n ' - 'retries 3\n ' - 'timeout queue 20s\n ' - 'timeout connect 5s\n ' - 'timeout http-request 1s\n ' - 'timeout http-keep-alive 5s\n ' - 'timeout client 30s\n ' - 'timeout server 30s\n ' - 'timeout check 5s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n' - ' mode http\n' - ' bind 1.2.3.4:8999\n' - ' bind 1.2.3.7:8999\n' - ' monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089\n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'option forwardfor\n ' - 'balance static-rr\n ' - 'option httpchk HEAD / HTTP/1.0\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' - } - } - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - assert gen_config_lines == exp_config_lines - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_haproxy_ssl(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': [ - '1.2.3.4', # simulate already assigned VIP - '1.2.3.1', # simulate interface IP - ] - } - }) - - # the ingress backend - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32", - ssl=True, - enable_stats=True, - monitor_ssl=True) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode http\n ' - 'log global\n ' - 'option httplog\n ' - 'option dontlognull\n ' - 'option http-server-close\n ' - 'option forwardfor except 127.0.0.0/8\n ' - 'option redispatch\n ' - 'retries 3\n ' - 'timeout queue 20s\n ' - 'timeout connect 5s\n ' - 'timeout http-request 1s\n ' - 'timeout http-keep-alive 5s\n ' - 'timeout client 30s\n ' - 'timeout server 30s\n ' - 'timeout check 5s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n ' - 'mode http\n ' - 'bind 1.2.3.4:8999 ssl crt /var/lib/haproxy/haproxy.pem\n ' - 'bind 1.2.3.7:8999 ssl crt /var/lib/haproxy/haproxy.pem\n ' - 'stats enable\n ' - 'stats uri /stats\n ' - 'stats refresh 10s\n ' - 'stats auth admin:12345\n ' - 'http-request use-service prometheus-exporter if { path /metrics }\n ' - 'monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089 ssl crt /var/lib/haproxy/haproxy.pem\n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'option forwardfor\n ' - 'balance static-rr\n ' - 'option httpchk HEAD / HTTP/1.0\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' - } - } - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - assert gen_config_lines == exp_config_lines - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_haproxy_with_different_stats_cert(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': [ - '1.2.3.4', # simulate already assigned VIP - '1.2.3.1', # simulate interface IP - ] - } - }) - - # the ingress backend - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32", - ssl=True, - enable_stats=True, - monitor_ssl=True, - monitor_cert_source='cephadm-signed' - ) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode http\n ' - 'log global\n ' - 'option httplog\n ' - 'option dontlognull\n ' - 'option http-server-close\n ' - 'option forwardfor except 127.0.0.0/8\n ' - 'option redispatch\n ' - 'retries 3\n ' - 'timeout queue 20s\n ' - 'timeout connect 5s\n ' - 'timeout http-request 1s\n ' - 'timeout http-keep-alive 5s\n ' - 'timeout client 30s\n ' - 'timeout server 30s\n ' - 'timeout check 5s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n ' - 'mode http\n ' - 'bind 1.2.3.4:8999 ssl crt /var/lib/haproxy/stats_haproxy.pem\n ' - 'bind 1.2.3.7:8999 ssl crt /var/lib/haproxy/stats_haproxy.pem\n ' - 'stats enable\n ' - 'stats uri /stats\n ' - 'stats refresh 10s\n ' - 'stats auth admin:12345\n ' - 'http-request use-service prometheus-exporter if { path /metrics }\n ' - 'monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089 ssl crt /var/lib/haproxy/haproxy.pem\n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'option forwardfor\n ' - 'balance static-rr\n ' - 'option httpchk HEAD / HTTP/1.0\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' - } - } - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - assert gen_config_lines == exp_config_lines - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_ingress_haproxy_monitor_ip(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': [ - '1.2.3.4', # simulate already assigned VIP - '1.2.3.1', # simulate interface IP - ] - } - }) - - # the ingress backend - s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1), - rgw_frontend_type='beast') - - ispec = IngressSpec(service_type='ingress', - service_id='test', - backend_service='rgw.foo', - frontend_port=8089, - monitor_port=8999, - monitor_user='admin', - monitor_password='12345', - keepalived_password='12345', - virtual_interface_networks=['1.2.3.0/24'], - virtual_ip="1.2.3.4/32", - enable_stats=True, - monitor_ip_addrs={'test': '1.2.3.1'}) - with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _: - # generate the haproxy conf based on the specified spec - haproxy_generated_conf = service_registry.get_service('ingress').haproxy_generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name())) - - haproxy_expected_conf = { - 'files': - { - 'haproxy.cfg': - '# This file is generated by cephadm.' - '\nglobal\n log ' - '127.0.0.1 local2\n ' - 'chroot /var/lib/haproxy\n ' - 'pidfile /var/lib/haproxy/haproxy.pid\n ' - 'maxconn 8000\n ' - 'daemon\n ' - 'stats socket /var/lib/haproxy/stats\n' - '\ndefaults\n ' - 'mode http\n ' - 'log global\n ' - 'option httplog\n ' - 'option dontlognull\n ' - 'option http-server-close\n ' - 'option forwardfor except 127.0.0.0/8\n ' - 'option redispatch\n ' - 'retries 3\n ' - 'timeout queue 20s\n ' - 'timeout connect 5s\n ' - 'timeout http-request 1s\n ' - 'timeout http-keep-alive 5s\n ' - 'timeout client 30s\n ' - 'timeout server 30s\n ' - 'timeout check 5s\n ' - 'maxconn 8000\n' - '\nfrontend stats\n ' - 'mode http\n ' - 'bind 1.2.3.1:8999\n ' - 'stats enable\n ' - 'stats uri /stats\n ' - 'stats refresh 10s\n ' - 'stats auth admin:12345\n ' - 'http-request use-service prometheus-exporter if { path /metrics }\n ' - 'monitor-uri /health\n' - '\nfrontend frontend\n ' - 'bind 1.2.3.4:8089\n ' - 'default_backend backend\n\n' - 'backend backend\n ' - 'option forwardfor\n ' - 'balance static-rr\n ' - 'option httpchk HEAD / HTTP/1.0\n ' - 'server ' - + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n' - } - } - - gen_config_lines = [line.rstrip() for line in haproxy_generated_conf[0]['files']['haproxy.cfg'].splitlines()] - exp_config_lines = [line.rstrip() for line in haproxy_expected_conf['files']['haproxy.cfg'].splitlines()] - - assert gen_config_lines == exp_config_lines - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.nfs.NFSService.fence_old_ranks", MagicMock()) - @patch("cephadm.services.nfs.NFSService.run_grace_tool", MagicMock()) - @patch("cephadm.services.nfs.NFSService.purge", MagicMock()) - @patch("cephadm.services.nfs.NFSService.create_rados_config_obj", MagicMock()) - def test_nfs_tls(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - with with_host(cephadm_module, 'test', addr='1.2.3.7'): - cephadm_module.cache.update_host_networks('test', { - '1.2.3.0/24': { - 'if0': ['1.2.3.1'] - } - }) - - nfs_spec = NFSServiceSpec(service_id="foo", placement=PlacementSpec(hosts=['test']), - ssl=True, ssl_cert=ceph_generated_cert, ssl_key=ceph_generated_key, - ssl_ca_cert=cephadm_root_ca, certificate_source='inline', tls_ktls=True, - tls_debug=True, tls_min_version='TLSv1.3', - tls_ciphers='ECDHE-ECDSA-AES256') - with with_service(cephadm_module, nfs_spec) as _: - nfs_generated_conf, _ = service_registry.get_service('nfs').generate_config( - CephadmDaemonDeploySpec(host='test', daemon_id='foo.test.0.0', service_name=nfs_spec.service_name())) - ganesha_conf = nfs_generated_conf['files']['ganesha.conf'] - expected_tls_block = ( - 'TLS_CONFIG{\n' - ' Enable_TLS = True;\n' - ' TLS_Cert_File = /etc/ganesha/tls/tls_cert.pem;\n' - ' TLS_Key_File = /etc/ganesha/tls/tls_key.pem;\n' - ' TLS_CA_File = /etc/ganesha/tls/tls_ca_cert.pem;\n' - ' TLS_Ciphers = "ECDHE-ECDSA-AES256";\n' - ' TLS_Min_Version = "TLSv1.3";\n' - ' Enable_KTLS = True;\n' - ' Enable_debug = True;\n' - '}\n' - ) - assert expected_tls_block in ganesha_conf - - -class TestCephFsMirror: - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, ServiceSpec('cephfs-mirror')): - cephadm_module.assert_issued_mon_command({ - 'prefix': 'mgr module enable', - 'module': 'mirroring' - }) - - -class TestJaeger: - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_jaeger_query(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - spec = TracingSpec(es_nodes="192.168.0.1:9200", service_type="jaeger-query") - config = {"elasticsearch_nodes": "http://192.168.0.1:9200"} - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'test', - "jaeger-query.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'jaeger-query.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [16686], - }, - "meta": { - 'service_name': 'jaeger-query', - 'ports': [16686], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_jaeger_collector_es_deploy(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - collector_spec = TracingSpec(service_type="jaeger-collector") - es_spec = TracingSpec(service_type="elasticsearch") - es_config = {} - - with with_host(cephadm_module, 'test'): - collector_config = { - "elasticsearch_nodes": f'http://{build_url(host=cephadm_module.inventory.get_addr("test"), port=9200).lstrip("/")}'} - with with_service(cephadm_module, es_spec): - _run_cephadm.assert_called_with( - "test", - "elasticsearch.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'elasticsearch.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9200], - }, - "meta": { - 'service_name': 'elasticsearch', - 'ports': [9200], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": es_config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - with with_service(cephadm_module, collector_spec): - _run_cephadm.assert_called_with( - "test", - "jaeger-collector.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'jaeger-collector.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [14250], - }, - "meta": { - 'service_name': 'jaeger-collector', - 'ports': [14250], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": collector_config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_jaeger_agent(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - collector_spec = TracingSpec(service_type="jaeger-collector", es_nodes="192.168.0.1:9200") - collector_config = {"elasticsearch_nodes": "http://192.168.0.1:9200"} - - agent_spec = TracingSpec(service_type="jaeger-agent") - agent_config = {"collector_nodes": "test:14250"} - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, collector_spec): - _run_cephadm.assert_called_with( - "test", - "jaeger-collector.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'jaeger-collector.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [14250], - }, - "meta": { - 'service_name': 'jaeger-collector', - 'ports': [14250], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": collector_config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - with with_service(cephadm_module, agent_spec): - _run_cephadm.assert_called_with( - "test", - "jaeger-agent.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'jaeger-agent.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [6799], - }, - "meta": { - 'service_name': 'jaeger-agent', - 'ports': [6799], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": agent_config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - -class TestAgent: - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None, fqdns=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_deploy_cephadm_agent(self, _run_cephadm, cephadm_module: CephadmOrchestrator): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - agent_spec = ServiceSpec(service_type="agent", placement=PlacementSpec(count=1)) - agent_config = {"agent.json": "{\"target_ip\": \"::1\", \"target_port\": 7150, \"refresh_period\": 20, \"listener_port\": 4721, \"host\": \"test\", \"device_enhanced_scan\": \"False\"}", "keyring": "[client.agent.test]\nkey = None\n", "root_cert.pem": f"{cephadm_root_ca}", "listener.crt": f"{ceph_generated_cert}", "listener.key": f"{ceph_generated_key}"} - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, agent_spec): - _run_cephadm.assert_called_with( - "test", - "agent.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps({ - "fsid": "fsid", - "name": 'agent.test', - "image": '', - "deploy_arguments": [], - "params": { - 'tcp_ports': [4721], - }, - "meta": { - 'service_name': 'agent', - 'ports': [4721], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": agent_config, - }), - error_ok=True, - use_current_daemon_image=False, - ) - - -class TestCustomContainer: - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_deploy_custom_container( - self, _run_cephadm, cephadm_module: CephadmOrchestrator - ): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - spec = CustomContainerSpec( - service_id='tsettinu', - image='quay.io/foobar/barbaz:latest', - entrypoint='/usr/local/bin/blat.sh', - ports=[9090], - ) - - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'test', - "container.tsettinu.test", - ['_orch', 'deploy'], - [], - stdin=json.dumps( - { - "fsid": "fsid", - "name": 'container.tsettinu.test', - "image": 'quay.io/foobar/barbaz:latest', - "deploy_arguments": [], - "params": { - 'tcp_ports': [9090], - }, - "meta": { - 'service_name': 'container.tsettinu', - 'ports': [], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - "config_blobs": { - "image": "quay.io/foobar/barbaz:latest", - "entrypoint": "/usr/local/bin/blat.sh", - "args": [], - "envs": [], - "volume_mounts": {}, - "privileged": False, - "ports": [9090], - "dirs": [], - "files": {}, - }, - } - ), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_deploy_custom_container_with_init_ctrs( - self, _run_cephadm, cephadm_module: CephadmOrchestrator - ): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - - spec = CustomContainerSpec( - service_id='tsettinu', - image='quay.io/foobar/barbaz:latest', - entrypoint='/usr/local/bin/blat.sh', - ports=[9090], - init_containers=[ - {'entrypoint': '/usr/local/bin/prepare.sh'}, - { - 'entrypoint': '/usr/local/bin/optimize.sh', - 'entrypoint_args': [ - '--timeout=5m', - '--cores=8', - {'argument': '--title=Alpha One'}, - ], - }, - ], - ) - - expected = { - 'fsid': 'fsid', - 'name': 'container.tsettinu.test', - 'image': 'quay.io/foobar/barbaz:latest', - 'deploy_arguments': [], - 'params': { - 'tcp_ports': [9090], - 'init_containers': [ - {'entrypoint': '/usr/local/bin/prepare.sh'}, - { - 'entrypoint': '/usr/local/bin/optimize.sh', - 'entrypoint_args': [ - '--timeout=5m', - '--cores=8', - '--title=Alpha One', - ], - }, - ], - }, - 'meta': { - 'service_name': 'container.tsettinu', - 'ports': [], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - 'init_containers': [ - {'entrypoint': '/usr/local/bin/prepare.sh'}, - { - 'entrypoint': '/usr/local/bin/optimize.sh', - 'entrypoint_args': [ - '--timeout=5m', - '--cores=8', - {'argument': '--title=Alpha One', 'split': False}, - ], - }, - ], - }, - 'config_blobs': { - 'image': 'quay.io/foobar/barbaz:latest', - 'entrypoint': '/usr/local/bin/blat.sh', - 'args': [], - 'envs': [], - 'volume_mounts': {}, - 'privileged': False, - 'ports': [9090], - 'dirs': [], - 'files': {}, - }, - } - with with_host(cephadm_module, 'test'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'test', - 'container.tsettinu.test', - ['_orch', 'deploy'], - [], - stdin=json.dumps(expected), - error_ok=True, - use_current_daemon_image=False, - ) - - -_SAMBA_METRICS_IMAGE = 'quay.io/samba.org/samba-metrics:devbuilds-centos-amd64' - - -class TestSMB: - @patch("cephadm.module.CephadmOrchestrator.get_unique_name") - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_deploy_smb( - self, _run_cephadm, _get_uname, cephadm_module: CephadmOrchestrator - ): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - _get_uname.return_value = 'tango.briskly' - - spec = SMBSpec( - cluster_id='foxtrot', - config_uri='rados://.smb/foxtrot/config.json', - ) - - expected = { - 'fsid': 'fsid', - 'name': 'smb.tango.briskly', - 'image': '', - 'deploy_arguments': [], - 'params': { - "tcp_ports": [445, 9922] - }, - 'meta': { - 'service_name': 'smb', - 'ports': [445, 9922], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - 'config_blobs': { - 'cluster_id': 'foxtrot', - 'features': [], - 'config_uri': 'rados://.smb/foxtrot/config.json', - 'config': '', - 'keyring': '[client.smb.config.tango.briskly]\nkey = None\n', - 'config_auth_entity': 'client.smb.config.tango.briskly', - 'metrics_image': _SAMBA_METRICS_IMAGE, - 'service_ports': { - 'smb': 445, - 'smbmetrics': 9922, - 'ctdb': 4379, - 'remote-control': 54445, - }, - }, - } - with with_host(cephadm_module, 'hostx'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'hostx', - 'smb.tango.briskly', - ['_orch', 'deploy'], - [], - stdin=json.dumps(expected), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.module.CephadmOrchestrator.get_unique_name") - @patch("cephadm.serve.CephadmServe._run_cephadm") - def test_deploy_smb_join_dns( - self, _run_cephadm, _get_uname, cephadm_module: CephadmOrchestrator - ): - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - _get_uname.return_value = 'tango.briskly' - - spec = SMBSpec( - cluster_id='foxtrot', - features=['domain'], - config_uri='rados://.smb/foxtrot/config2.json', - join_sources=[ - 'rados://.smb/foxtrot/join1.json', - 'rados:mon-config-key:smb/config/foxtrot/join2.json', - ], - custom_dns=['10.8.88.103'], - include_ceph_users=[ - 'client.smb.fs.cephfs.share1', - 'client.smb.fs.cephfs.share2', - 'client.smb.fs.fs2.share3', - ], - ) - - expected = { - 'fsid': 'fsid', - 'name': 'smb.tango.briskly', - 'image': '', - 'deploy_arguments': [], - 'params': { - 'tcp_ports': [445, 9922] - }, - 'meta': { - 'service_name': 'smb', - 'ports': [445, 9922], - 'ip': None, - 'deployed_by': [], - 'rank': None, - 'rank_generation': None, - 'extra_container_args': None, - 'extra_entrypoint_args': None, - }, - 'config_blobs': { - 'cluster_id': 'foxtrot', - 'features': ['domain'], - 'config_uri': 'rados://.smb/foxtrot/config2.json', - 'join_sources': [ - 'rados://.smb/foxtrot/join1.json', - 'rados:mon-config-key:smb/config/foxtrot/join2.json', - ], - 'custom_dns': ['10.8.88.103'], - 'config': '', - 'keyring': ( - '[client.smb.config.tango.briskly]\nkey = None\n\n' - '[client.smb.fs.cephfs.share1]\nkey = None\n\n' - '[client.smb.fs.cephfs.share2]\nkey = None\n\n' - '[client.smb.fs.fs2.share3]\nkey = None\n' - ), - 'config_auth_entity': 'client.smb.config.tango.briskly', - 'metrics_image': _SAMBA_METRICS_IMAGE, - 'service_ports': { - 'smb': 445, - 'smbmetrics': 9922, - 'ctdb': 4379, - 'remote-control': 54445, - }, - }, - } - with with_host(cephadm_module, 'hostx'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'hostx', - 'smb.tango.briskly', - ['_orch', 'deploy'], - [], - stdin=json.dumps(expected), - error_ok=True, - use_current_daemon_image=False, - ) - - -class TestMgmtGateway: - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_endpoints") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_discovery_endpoints") - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_self_signed_certificates_with_label", - lambda instance, svc_spec, dspec, label, ip: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.mgmt_gateway.get_dashboard_endpoints", lambda _: (["ceph-node-2:8443", "ceph-node-2:8443"], "https")) - def test_mgmt_gateway_config_no_auth(self, - get_service_discovery_endpoints_mock: List[str], - get_service_endpoints_mock: List[str], - _run_cephadm, - cephadm_module: CephadmOrchestrator): - - def get_services_endpoints(name): - if name == 'prometheus': - return ["192.168.100.100:9095", "192.168.100.101:9095"] - elif name == 'grafana': - return ["ceph-node-2:3000", "ceph-node-2:3000"] - elif name == 'alertmanager': - return ["192.168.100.100:9093", "192.168.100.102:9093"] - return [] - - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - get_service_endpoints_mock.side_effect = get_services_endpoints - get_service_discovery_endpoints_mock.side_effect = lambda: ["ceph-node-0:8765", "ceph-node-2:8765"] - - server_port = 5555 - spec = MgmtGatewaySpec(port=server_port, - ssl_cert=ceph_generated_cert, - ssl_key=ceph_generated_key) - - expected = { - "fsid": "fsid", - "name": "mgmt-gateway.ceph-node", - "image": "", - "deploy_arguments": [], - "params": {"tcp_ports": [server_port]}, - "meta": { - "service_name": "mgmt-gateway", - "ports": [server_port], - "ip": None, - "deployed_by": [], - "rank": None, - "rank_generation": None, - "extra_container_args": None, - "extra_entrypoint_args": None - }, - "config_blobs": { - "files": { - "nginx.conf": dedent(""" - # This file is generated by cephadm. - worker_rlimit_nofile 8192; - - events { - worker_connections 4096; - } - - http { - - #access_log /dev/stdout; - error_log /dev/stderr warn; - client_header_buffer_size 32K; - large_client_header_buffers 4 32k; - proxy_busy_buffers_size 512k; - proxy_buffers 4 512k; - proxy_buffer_size 256K; - proxy_headers_hash_max_size 1024; - proxy_headers_hash_bucket_size 128; - - - upstream service_discovery_servers { - server ceph-node-0:8765; - server ceph-node-2:8765; - } - - upstream dashboard_servers { - server ceph-node-2:8443; - server ceph-node-2:8443; - } - - upstream grafana_servers { - server ceph-node-2:3000; - server ceph-node-2:3000; - } - - upstream prometheus_servers { - ip_hash; - server 192.168.100.100:9095; - server 192.168.100.101:9095; - } - - upstream alertmanager_servers { - server 192.168.100.100:9093; - server 192.168.100.102:9093; - } - - include /etc/nginx_external_server.conf; - include /etc/nginx_internal_server.conf; - }"""), - "nginx_external_server.conf": dedent(""" - server { - listen 5555 ssl; - listen [::]:5555 ssl; - ssl_certificate /etc/nginx/ssl/nginx.crt; - ssl_certificate_key /etc/nginx/ssl/nginx.key; - ssl_protocols TLSv1.3; - # from: https://ssl-config.mozilla.org/#server=nginx - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; - - # Only return Nginx in server header, no extra info will be provided - server_tokens off; - - # Perfect Forward Secrecy(PFS) is frequently compromised without this - ssl_prefer_server_ciphers on; - - # Enable SSL session caching for improved performance - ssl_session_tickets off; - ssl_session_timeout 1d; - ssl_session_cache shared:SSL:10m; - - # OCSP stapling - ssl_stapling on; - ssl_stapling_verify on; - resolver_timeout 5s; - - # Security headers - ## X-Content-Type-Options: avoid MIME type sniffing - add_header X-Content-Type-Options nosniff; - ## Strict Transport Security (HSTS): Yes - add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload"; - ## Enables the Cross-site scripting (XSS) filter in browsers. - add_header X-XSS-Protection "1; mode=block"; - ## Content-Security-Policy (CSP): FIXME - # add_header Content-Security-Policy "default-src 'self'; script-src 'self'; object-src 'none'; base-uri 'none'; require-trusted-types-for 'script'; frame-ancestors 'self';"; - - - location / { - proxy_pass https://dashboard_servers; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - } - - location /grafana { - proxy_pass https://grafana_servers; - # clear any Authorization header as Prometheus and Alertmanager are using basic-auth browser - # will send this header if Grafana is running on the same node as one of those services - proxy_set_header Authorization ""; - proxy_buffering off; - } - - location /prometheus { - proxy_pass https://prometheus_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - } - - location /alertmanager { - proxy_pass https://alertmanager_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - } - }"""), - "nginx_internal_server.conf": dedent(""" - server { - ssl_client_certificate /etc/nginx/ssl/ca.crt; - ssl_verify_client on; - - listen 29443 ssl; - listen [::]:29443 ssl; - ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - ssl_protocols TLSv1.3; - # from: https://ssl-config.mozilla.org/#server=nginx - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; - ssl_prefer_server_ciphers on; - - location /internal/sd { - rewrite ^/internal/(.*) /$1 break; - proxy_pass https://service_discovery_servers; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - } - - location /internal/dashboard { - rewrite ^/internal/dashboard/(.*) /$1 break; - proxy_pass https://dashboard_servers; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - } - - location /internal/grafana { - rewrite ^/internal/grafana/(.*) /$1 break; - proxy_pass https://grafana_servers; - } - - location /internal/prometheus { - rewrite ^/internal/prometheus/(.*) /prometheus/$1 break; - proxy_pass https://prometheus_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - } - - location /internal/alertmanager { - rewrite ^/internal/alertmanager/(.*) /alertmanager/$1 break; - proxy_pass https://alertmanager_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - } - }"""), - "nginx_internal.crt": f"{ceph_generated_cert}", - "nginx_internal.key": f"{ceph_generated_key}", - "ca.crt": f"{cephadm_root_ca}", - "nginx.crt": f"{ceph_generated_cert}", - "nginx.key": f"{ceph_generated_key}", - } - } - } - - with with_host(cephadm_module, 'ceph-node'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'ceph-node', - 'mgmt-gateway.ceph-node', - ['_orch', 'deploy'], - [], - stdin=json.dumps(expected), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_endpoints") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_discovery_endpoints") - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_self_signed_certificates_with_label", - lambda instance, svc_spec, dspec, label, ip: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.mgmt_gateway.get_dashboard_endpoints", lambda _: (["ceph-node-2:8443", "ceph-node-2:8443"], "https")) - def test_mgmt_gateway_config_with_auth(self, - get_service_discovery_endpoints_mock: List[str], - get_service_endpoints_mock: List[str], - _run_cephadm, - cephadm_module: CephadmOrchestrator): - - def get_services_endpoints(name): - if name == 'prometheus': - return ["192.168.100.100:9095", "192.168.100.101:9095"] - elif name == 'grafana': - return ["ceph-node-2:3000", "ceph-node-2:3000"] - elif name == 'alertmanager': - return ["192.168.100.100:9093", "192.168.100.102:9093"] - elif name == 'oauth2-proxy': - return ["192.168.100.101:4180", "192.168.100.102:4180"] - return [] - - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - get_service_endpoints_mock.side_effect = get_services_endpoints - get_service_discovery_endpoints_mock.side_effect = lambda: ["ceph-node-0:8765", "ceph-node-2:8765"] - - server_port = 5555 - spec = MgmtGatewaySpec(port=server_port, - ssl_cert=ceph_generated_cert, - ssl_key=ceph_generated_key, - enable_auth=True) - - expected = { - "fsid": "fsid", - "name": "mgmt-gateway.ceph-node", - "image": "", - "deploy_arguments": [], - "params": {"tcp_ports": [server_port]}, - "meta": { - "service_name": "mgmt-gateway", - "ports": [server_port], - "ip": None, - "deployed_by": [], - "rank": None, - "rank_generation": None, - "extra_container_args": None, - "extra_entrypoint_args": None - }, - "config_blobs": { - "files": { - "nginx.conf": dedent(""" - # This file is generated by cephadm. - worker_rlimit_nofile 8192; - - events { - worker_connections 4096; - } - - http { - - #access_log /dev/stdout; - error_log /dev/stderr warn; - client_header_buffer_size 32K; - large_client_header_buffers 4 32k; - proxy_busy_buffers_size 512k; - proxy_buffers 4 512k; - proxy_buffer_size 256K; - proxy_headers_hash_max_size 1024; - proxy_headers_hash_bucket_size 128; - - upstream oauth2_proxy_servers { - server 192.168.100.101:4180; - server 192.168.100.102:4180; - } - - upstream service_discovery_servers { - server ceph-node-0:8765; - server ceph-node-2:8765; - } - - upstream dashboard_servers { - server ceph-node-2:8443; - server ceph-node-2:8443; - } - - upstream grafana_servers { - server ceph-node-2:3000; - server ceph-node-2:3000; - } - - upstream prometheus_servers { - ip_hash; - server 192.168.100.100:9095; - server 192.168.100.101:9095; - } - - upstream alertmanager_servers { - server 192.168.100.100:9093; - server 192.168.100.102:9093; - } - - include /etc/nginx_external_server.conf; - include /etc/nginx_internal_server.conf; - }"""), - "nginx_external_server.conf": dedent(""" - server { - listen 5555 ssl; - listen [::]:5555 ssl; - ssl_certificate /etc/nginx/ssl/nginx.crt; - ssl_certificate_key /etc/nginx/ssl/nginx.key; - ssl_protocols TLSv1.3; - # from: https://ssl-config.mozilla.org/#server=nginx - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; - - # Only return Nginx in server header, no extra info will be provided - server_tokens off; - - # Perfect Forward Secrecy(PFS) is frequently compromised without this - ssl_prefer_server_ciphers on; - - # Enable SSL session caching for improved performance - ssl_session_tickets off; - ssl_session_timeout 1d; - ssl_session_cache shared:SSL:10m; - - # OCSP stapling - ssl_stapling on; - ssl_stapling_verify on; - resolver_timeout 5s; - - # Security headers - ## X-Content-Type-Options: avoid MIME type sniffing - add_header X-Content-Type-Options nosniff; - ## Strict Transport Security (HSTS): Yes - add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload"; - ## Enables the Cross-site scripting (XSS) filter in browsers. - add_header X-XSS-Protection "1; mode=block"; - ## Content-Security-Policy (CSP): FIXME - # add_header Content-Security-Policy "default-src 'self'; script-src 'self'; object-src 'none'; base-uri 'none'; require-trusted-types-for 'script'; frame-ancestors 'self';"; - - location /oauth2/ { - proxy_pass https://oauth2_proxy_servers; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; - # Check for original-uri header - proxy_set_header X-Auth-Request-Redirect $scheme://$host$request_uri; - } - - location = /oauth2/auth { - internal; - proxy_pass https://oauth2_proxy_servers; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; - # nginx auth_request includes headers but not body - proxy_set_header Content-Length ""; - proxy_pass_request_body off; - } - - location / { - proxy_pass https://dashboard_servers; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - auth_request /oauth2/auth; - error_page 401 = /oauth2/sign_in; - - auth_request_set $email $upstream_http_x_auth_request_email; - proxy_set_header X-Email $email; - - auth_request_set $groups $upstream_http_x_auth_request_groups; - proxy_set_header X-User-Groups $groups; - - auth_request_set $user $upstream_http_x_auth_request_user; - proxy_set_header X-User $user; - - auth_request_set $token $upstream_http_x_auth_request_access_token; - proxy_set_header X-Access-Token $token; - - auth_request_set $auth_cookie $upstream_http_set_cookie; - add_header Set-Cookie $auth_cookie; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host:80; - proxy_set_header X-Forwarded-Port 80; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Forwarded-Groups $groups; - - proxy_http_version 1.1; - - proxy_set_header X-Forwarded-Proto "https"; - proxy_ssl_verify off; - } - - location /grafana { - proxy_pass https://grafana_servers; - # clear any Authorization header as Prometheus and Alertmanager are using basic-auth browser - # will send this header if Grafana is running on the same node as one of those services - proxy_set_header Authorization ""; - proxy_buffering off; - auth_request /oauth2/auth; - error_page 401 = /oauth2/sign_in; - - proxy_set_header X-Original-URI "/"; - - auth_request_set $user $upstream_http_x_auth_request_user; - auth_request_set $email $upstream_http_x_auth_request_email; - proxy_set_header X-WEBAUTH-USER $user; - proxy_set_header X-WEBAUTH-EMAIL $email; - - # Pass role header to Grafana - proxy_set_header X-WEBAUTH-ROLE $http_x_auth_request_role; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - auth_request_set $auth_cookie $upstream_http_set_cookie; - add_header Set-Cookie $auth_cookie; - - proxy_set_header X-Forwarded-Proto $scheme; - } - - location /prometheus { - proxy_pass https://prometheus_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - auth_request /oauth2/auth; - error_page 401 = /oauth2/sign_in; - - auth_request_set $user $upstream_http_x_auth_request_user; - auth_request_set $email $upstream_http_x_auth_request_email; - proxy_set_header X-User $user; - proxy_set_header X-Email $email; - - auth_request_set $auth_cookie $upstream_http_set_cookie; - add_header Set-Cookie $auth_cookie; - } - - location /alertmanager { - proxy_pass https://alertmanager_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - auth_request /oauth2/auth; - error_page 401 = /oauth2/sign_in; - - auth_request_set $user $upstream_http_x_auth_request_user; - auth_request_set $email $upstream_http_x_auth_request_email; - proxy_set_header X-User $user; - proxy_set_header X-Email $email; - - auth_request_set $auth_cookie $upstream_http_set_cookie; - add_header Set-Cookie $auth_cookie; - } - }"""), - "nginx_internal_server.conf": dedent(""" - server { - ssl_client_certificate /etc/nginx/ssl/ca.crt; - ssl_verify_client on; - - listen 29443 ssl; - listen [::]:29443 ssl; - ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - ssl_protocols TLSv1.3; - # from: https://ssl-config.mozilla.org/#server=nginx - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; - ssl_prefer_server_ciphers on; - - location /internal/sd { - rewrite ^/internal/(.*) /$1 break; - proxy_pass https://service_discovery_servers; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - } - - location /internal/dashboard { - rewrite ^/internal/dashboard/(.*) /$1 break; - proxy_pass https://dashboard_servers; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - } - - location /internal/grafana { - rewrite ^/internal/grafana/(.*) /$1 break; - proxy_pass https://grafana_servers; - } - - location /internal/prometheus { - rewrite ^/internal/prometheus/(.*) /prometheus/$1 break; - proxy_pass https://prometheus_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - } - - location /internal/alertmanager { - rewrite ^/internal/alertmanager/(.*) /alertmanager/$1 break; - proxy_pass https://alertmanager_servers; - - proxy_ssl_certificate /etc/nginx/ssl/nginx_internal.crt; - proxy_ssl_certificate_key /etc/nginx/ssl/nginx_internal.key; - proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - } - }"""), - "nginx_internal.crt": f"{ceph_generated_cert}", - "nginx_internal.key": f"{ceph_generated_key}", - "ca.crt": f"{cephadm_root_ca}", - "nginx.crt": f"{ceph_generated_cert}", - "nginx.key": f"{ceph_generated_key}", - } - } - } - - with with_host(cephadm_module, 'ceph-node'): - with with_service(cephadm_module, spec): - _run_cephadm.assert_called_with( - 'ceph-node', - 'mgmt-gateway.ceph-node', - ['_orch', 'deploy'], - [], - stdin=json.dumps(expected), - error_ok=True, - use_current_daemon_image=False, - ) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_endpoints") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_discovery_endpoints") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_self_signed_certificates_with_label") - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.mgmt_gateway.get_dashboard_endpoints", - lambda _: (["ceph-node-2:8443", "ceph-node-2:8443"], "https")) - def test_mgmt_gateway_internal_cert_san_includes_vip( - self, - get_self_signed_mock, - get_service_discovery_endpoints_mock, - get_service_endpoints_mock, - _run_cephadm, - cephadm_module: CephadmOrchestrator, - ): - vip = "10.0.0.200" - - def get_services_endpoints(name): - if name == 'prometheus': - return ["192.168.100.100:9095", "192.168.100.101:9095"] - if name == 'grafana': - return ["ceph-node-2:3000", "ceph-node-2:3000"] - if name == 'alertmanager': - return ["192.168.100.100:9093", "192.168.100.102:9093"] - if name == 'oauth2-proxy': - return [] - return [] - - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - get_service_endpoints_mock.side_effect = get_services_endpoints - get_service_discovery_endpoints_mock.return_value = ["ceph-node-0:8765", "ceph-node-2:8765"] - get_self_signed_mock.return_value = TLSCredentials(ceph_generated_cert, ceph_generated_key) - - server_port = 5555 - spec = MgmtGatewaySpec( - port=server_port, - virtual_ip=vip, # HA mode - ssl_cert=ceph_generated_cert, - ssl_key=ceph_generated_key, - ) - - with with_host(cephadm_module, 'ceph-node'): - with with_service(cephadm_module, spec): - # Ensure VIP was used when minting the internal cert (so it goes into SANs) - # get_self_signed_certificates_with_label(svc_spec, daemon_spec, label, ip) - args, _ = get_self_signed_mock.call_args - assert args[2] == 'internal' - assert args[3] == vip - deployed = json.loads(_run_cephadm.call_args.kwargs['stdin']) - assert deployed['config_blobs']['files']['nginx_internal.crt'] == ceph_generated_cert - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_endpoints") - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_self_signed_certificates_with_label", - lambda instance, svc_spec, dspec, label, ip: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.mgmt_gateway.get_dashboard_endpoints", lambda _: (["ceph-node-2:8443", "ceph-node-2:8443"], "https")) - def test_oauth2_proxy_service(self, get_service_endpoints_mock, _run_cephadm, cephadm_module): - self.oauth2_proxy_service_common(get_service_endpoints_mock, _run_cephadm, cephadm_module, virtual_ip=None) - - @patch("cephadm.serve.CephadmServe._run_cephadm") - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_service_endpoints") - @patch("cephadm.services.cephadmservice.CephadmService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.services.oauth2_proxy.OAuth2ProxyService.get_certificates", - lambda instance, dspec, ips=None: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.services.mgmt_gateway.MgmtGatewayService.get_self_signed_certificates_with_label", - lambda instance, svc_spec, dspec, label, ip: TLSCredentials(ceph_generated_cert, ceph_generated_key)) - @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1') - @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: cephadm_root_ca) - @patch("cephadm.services.mgmt_gateway.get_dashboard_endpoints", lambda _: (["ceph-node-2:8443", "ceph-node-2:8443"], "https")) - def test_oauth2_proxy_service_with_ha(self, get_service_endpoints_mock, _run_cephadm, cephadm_module): - self.oauth2_proxy_service_common(get_service_endpoints_mock, _run_cephadm, cephadm_module, virtual_ip="192.168.100.200") - - def oauth2_proxy_service_common(self, get_service_endpoints_mock, _run_cephadm, cephadm_module: CephadmOrchestrator, virtual_ip=None): - def get_services_endpoints(name): - if name == 'prometheus': - return ["192.168.100.100:9095", "192.168.100.101:9095"] - elif name == 'grafana': - return ["ceph-node-2:3000", "ceph-node-2:3000"] - elif name == 'alertmanager': - return ["192.168.100.100:9093", "192.168.100.102:9093"] - return [] - - _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) - get_service_endpoints_mock.side_effect = get_services_endpoints - - server_port = 5555 - mgmt_gw_spec = MgmtGatewaySpec(port=server_port, - ssl_cert=ceph_generated_cert, - ssl_key=ceph_generated_key, - enable_auth=True, - virtual_ip=virtual_ip) - - allowed_domain = '192.168.100.1:8080' - oauth2_spec = OAuth2ProxySpec(provider_display_name='my_idp_provider', - client_id='my_client_id', - client_secret='my_client_secret', - oidc_issuer_url='http://192.168.10.10:8888/dex', - cookie_secret='kbAEM9opAmuHskQvt0AW8oeJRaOM2BYy5Loba0kZ0SQ=', - ssl_cert=ceph_generated_cert, - ssl_key=ceph_generated_key, - allowlist_domains=[allowed_domain]) - - whitelist_domains = f"{allowed_domain},1::4,ceph-node" if virtual_ip is None else f"{allowed_domain},{virtual_ip},1::4,ceph-node" - redirect_url = f"https://{virtual_ip if virtual_ip else 'host_fqdn'}:5555/oauth2/callback" - expected = { - "fsid": "fsid", - "name": "oauth2-proxy.ceph-node", - "image": "", - "deploy_arguments": [], - "params": {"tcp_ports": [4180]}, - "meta": { - "service_name": "oauth2-proxy", - "ports": [4180], - "ip": None, - "deployed_by": [], - "rank": None, - "rank_generation": None, - "extra_container_args": None, - "extra_entrypoint_args": None - }, - "config_blobs": { - "files": { - "oauth2-proxy.conf": dedent(f""" - # Listen on port 4180 for incoming HTTP traffic. - https_address= "0.0.0.0:4180" - - skip_provider_button= true - skip_jwt_bearer_tokens= true - - # OIDC provider configuration. - provider= "oidc" - provider_display_name= "my_idp_provider" - client_id= "my_client_id" - client_secret= "my_client_secret" - oidc_issuer_url= "http://192.168.10.10:8888/dex" - redirect_url= "{redirect_url}" - - ssl_insecure_skip_verify=true - - # following configuration is needed to avoid getting Forbidden - # when using chrome like browsers as they handle 3rd party cookies - # more strictly than Firefox - cookie_samesite= "none" - cookie_secure= true - cookie_expire= "5h" - cookie_refresh= "2h" - - pass_access_token= true - pass_authorization_header= true - pass_basic_auth= true - pass_user_headers= true - set_xauthrequest= true - - # Secret value for encrypting cookies. - cookie_secret= "kbAEM9opAmuHskQvt0AW8oeJRaOM2BYy5Loba0kZ0SQ=" - email_domains= "*" - whitelist_domains= "{whitelist_domains}\""""), - "oauth2-proxy.crt": f"{ceph_generated_cert}", - "oauth2-proxy.key": f"{ceph_generated_key}", - } - } - } - - with with_host(cephadm_module, 'ceph-node'): - with with_service(cephadm_module, mgmt_gw_spec) as _, with_service(cephadm_module, oauth2_spec): - _run_cephadm.assert_called_with( - 'ceph-node', - 'oauth2-proxy.ceph-node', - ['_orch', 'deploy'], - [], - stdin=json.dumps(expected), - error_ok=True, - use_current_daemon_image=False, - )