From f6d0fa2f043df8c0f5b4f9b11026fcddcd734409 Mon Sep 17 00:00:00 2001 From: Abhishek Desai Date: Tue, 19 Aug 2025 12:41:19 +0530 Subject: [PATCH] mgr/dashboard : Skip calls until secure_monitoring_stack is enabled fixes : https://tracker.ceph.com/issues/72635 Signed-off-by: Abhishek Desai (cherry picked from commit 42ce56f4b96a46f01d2078132ced40182aa30d68) --- .../mgr/dashboard/controllers/prometheus.py | 102 +++++++++++++++--- .../mgr/dashboard/services/orchestrator.py | 14 +++ src/pybind/mgr/dashboard/settings.py | 1 + .../mgr/dashboard/tests/test_prometheus.py | 57 ++++++++-- 4 files changed, 146 insertions(+), 28 deletions(-) diff --git a/src/pybind/mgr/dashboard/controllers/prometheus.py b/src/pybind/mgr/dashboard/controllers/prometheus.py index c00d8c70e63..20f2fa1a764 100644 --- a/src/pybind/mgr/dashboard/controllers/prometheus.py +++ b/src/pybind/mgr/dashboard/controllers/prometheus.py @@ -2,7 +2,9 @@ import json import os import tempfile +import time from datetime import datetime +from typing import NamedTuple, Optional import requests @@ -10,11 +12,20 @@ from .. import mgr from ..exceptions import DashboardException from ..security import Scope from ..services import ceph_service +from ..services.orchestrator import OrchClient from ..services.settings import SettingsService from ..settings import Options, Settings from . import APIDoc, APIRouter, BaseController, Endpoint, RESTController, Router, UIRouter +class Credentials(NamedTuple): + user: str + password: str + ca_cert_file: Optional[str] + cert_file: Optional[str] + pkey_file: Optional[str] + + @Router('/api/prometheus_receiver', secure=False) class PrometheusReceiver(BaseController): """ @@ -31,6 +42,10 @@ class PrometheusReceiver(BaseController): class PrometheusRESTController(RESTController): + # Cache for credentials for 1-minute + _credentials_cache = {} + _cache_timestamp = {} + def close_unlink_files(self, files): # type (List[str]) valid_entries = [f for f in files if f is not None] @@ -38,6 +53,37 @@ class PrometheusRESTController(RESTController): f.close() os.unlink(f.name) + def _is_cache_valid(self, module_name): + """Check if cached credentials are still valid (1 minute)""" + if module_name not in self._cache_timestamp: + return False + current_time = time.time() + return ((current_time - self._cache_timestamp[module_name]) + < Settings.PROM_ALERT_CREDENTIAL_CACHE_TTL) + + def _get_cached_credentials(self, module_name): + """ + Get cached credentials if they exist and are valid + Clears the cached credentials if invalid + """ + if self._is_cache_valid(module_name): + return self._credentials_cache.get(module_name) + old_creds = self._credentials_cache.get(module_name) + if old_creds: + self.close_unlink_files([ + old_creds.ca_cert_file, + old_creds.cert_file, + old_creds.pkey_file + ]) + self._credentials_cache.pop(module_name, None) + self._cache_timestamp.pop(module_name, None) + return None + + def _cache_credentials(self, module_name, credentials): + """Cache credentials with current timestamp""" + self._credentials_cache[module_name] = credentials + self._cache_timestamp[module_name] = time.time() + def prometheus_proxy(self, method, path, params=None, payload=None): # type (str, str, dict, dict) user, password, ca_cert_file, cert_file, key_file = self.get_access_info('prometheus') @@ -47,7 +93,6 @@ class PrometheusRESTController(RESTController): method, path, 'Prometheus', params, payload, user=user, password=password, verify=verify, cert=cert) - self.close_unlink_files([ca_cert_file, cert_file, key_file]) return response def alert_proxy(self, method, path, params=None, payload=None): @@ -59,7 +104,6 @@ class PrometheusRESTController(RESTController): method, path, 'Alertmanager', params, payload, user=user, password=password, verify=verify, cert=cert, is_alertmanager=True) - self.close_unlink_files([ca_cert_file, cert_file, key_file]) return response def get_access_info(self, module_name): @@ -72,29 +116,53 @@ class PrometheusRESTController(RESTController): tmp_file = tempfile.NamedTemporaryFile(delete=False) tmp_file.write(content.encode('utf-8')) tmp_file.flush() # tmp_file must not be gc'ed + tmp_file.close() return tmp_file + orch_backend = mgr.get_module_option_ex('orchestrator', 'orchestrator') + is_cephadm = orch_backend == 'cephadm' + if module_name not in ['prometheus', 'alertmanager']: - raise DashboardException(f'Invalid module name {module_name}', component='prometheus') + raise DashboardException(f'Invalid module name {module_name}', + coFalsemponent='prometheus') + user = None password = None cert_file = None pkey_file = None ca_cert_file = None - orch_backend = mgr.get_module_option_ex('orchestrator', 'orchestrator') - if orch_backend == 'cephadm': - cmd = {'prefix': f'orch {module_name} get-credentials'} - ret, out, _ = mgr.mon_command(cmd) - if ret == 0 and out is not None: - access_info = json.loads(out) - if access_info: - user = access_info['user'] - password = access_info['password'] - ca_cert_file = write_to_tmp_file(access_info['certificate']) - cert_file = write_to_tmp_file(mgr.get_localized_store("crt")) - pkey_file = write_to_tmp_file(mgr.get_localized_store("key")) - - return user, password, ca_cert_file, cert_file, pkey_file + + if not is_cephadm: + return Credentials(user, password, ca_cert_file, cert_file, pkey_file) + + cached_creds = self._get_cached_credentials(module_name) + if cached_creds: + return cached_creds + + secure_monitoring_stack = mgr.get_module_option_ex('cephadm', 'secure_monitoring_stack') + if not secure_monitoring_stack: + return Credentials(user, password, ca_cert_file, cert_file, pkey_file) + orch_client = OrchClient.instance() + if orch_client.available(): + if module_name == 'prometheus': + access_info = orch_client.monitoring.get_prometheus_access_info() + elif module_name == 'alertmanager': + access_info = orch_client.monitoring.get_alertmanager_access_info() + else: + access_info = None + if access_info: + user = access_info.get('user') + password = access_info.get('password') + ca_cert_file = write_to_tmp_file(access_info.get('certificate')) + cert_file = write_to_tmp_file(mgr.get_localized_store("crt")) + pkey_file = write_to_tmp_file(mgr.get_localized_store("key")) + # Cache the credentials + self._cache_credentials( + module_name, + Credentials(user, password, ca_cert_file, cert_file, pkey_file) + ) + + return Credentials(user, password, ca_cert_file, cert_file, pkey_file) def _get_api_url(self, host, version='v1'): return f'{host.rstrip("/")}/api/{version}' diff --git a/src/pybind/mgr/dashboard/services/orchestrator.py b/src/pybind/mgr/dashboard/services/orchestrator.py index 38859167568..22190f45fad 100644 --- a/src/pybind/mgr/dashboard/services/orchestrator.py +++ b/src/pybind/mgr/dashboard/services/orchestrator.py @@ -222,6 +222,19 @@ class CertStoreManager(ResourceManager): no_exception_when_missing=ignore_missing_exception) +class MonitoringManager(ResourceManager): + + @wait_api_result + def get_prometheus_access_info(self) -> Dict[str, str]: + """Get Prometheus access information""" + return self.api.get_prometheus_access_info() + + @wait_api_result + def get_alertmanager_access_info(self) -> Dict[str, str]: + """Get Alertmanager access information""" + return self.api.get_alertmanager_access_info() + + class OrchClient(object): _instance = None @@ -244,6 +257,7 @@ class OrchClient(object): self.upgrades = UpgradeManager(self.api) self.hardware = HardwareManager(self.api) self.cert_store = CertStoreManager(self.api) + self.monitoring = MonitoringManager(self.api) def available(self, features: Optional[List[str]] = None) -> bool: available = self.status()['available'] diff --git a/src/pybind/mgr/dashboard/settings.py b/src/pybind/mgr/dashboard/settings.py index 9cb1e3de289..d81213ad39a 100644 --- a/src/pybind/mgr/dashboard/settings.py +++ b/src/pybind/mgr/dashboard/settings.py @@ -88,6 +88,7 @@ class Options(object): PROMETHEUS_API_SSL_VERIFY = Setting(True, [bool]) ALERTMANAGER_API_HOST = Setting('', [str]) ALERTMANAGER_API_SSL_VERIFY = Setting(True, [bool]) + PROM_ALERT_CREDENTIAL_CACHE_TTL = Setting(60, [int]) # iSCSI management settings ISCSI_API_SSL_VERIFICATION = Setting(True, [bool]) diff --git a/src/pybind/mgr/dashboard/tests/test_prometheus.py b/src/pybind/mgr/dashboard/tests/test_prometheus.py index 7f795a47450..f50fe262109 100644 --- a/src/pybind/mgr/dashboard/tests/test_prometheus.py +++ b/src/pybind/mgr/dashboard/tests/test_prometheus.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- # pylint: disable=protected-access try: - from mock import patch + from mock import Mock, patch except ImportError: - from unittest.mock import patch + from unittest.mock import Mock, patch + +from requests import Response from .. import mgr from ..controllers.prometheus import Prometheus, PrometheusNotifications, PrometheusReceiver @@ -27,20 +29,53 @@ class PrometheusControllerTest(ControllerTestCase): cls.setup_controllers([Prometheus, PrometheusNotifications, PrometheusReceiver]) @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", return_value='cephadm') - @patch("dashboard.controllers.prometheus.mgr.mon_command", return_value=(1, {}, None)) + @patch('dashboard.controllers.prometheus.PrometheusRESTController.balancer_status', + return_value={'active': False, 'no_optimization_needed': False}) + @patch('dashboard.controllers.prometheus.mgr.get_localized_store', return_value=None) + @patch('dashboard.services.orchestrator.OrchClient.instance') + @patch('dashboard.services.orchestrator.OrchClient.status', return_value={'available': True}) + @patch('dashboard.services.orchestrator.OrchClient.available', return_value=True) @patch('requests.request') - def test_rules_cephadm(self, mock_request, mock_mon_command, mock_get_module_option_ex): + def test_rules_cephadm(self, mock_request, _mock_available, _mock_status, mock_instance, + _mock_get_localized_store, _mock_balancer_status, + mock_get_module_option_ex): + # in this test we use: # in the first call to get_module_option_ex we return 'cephadm' as backend # in the second call we return 'True' for 'secure_monitoring_stack' option - mock_get_module_option_ex.side_effect = lambda module, key, default=None: 'cephadm' \ - if module == 'orchestrator' else True + def _opt(module, key, default=None): + if module == 'orchestrator' and key == 'orchestrator': + return 'cephadm' + if module == 'cephadm' and key == 'secure_monitoring_stack': + return True + return default + mock_get_module_option_ex.side_effect = _opt + + # OrchClient.instance().monitoring.get_prometheus_access_info() + fake_orch = Mock() + fake_orch.monitoring.get_prometheus_access_info.return_value = { + 'user': None, + 'password': None, + 'certificate': None, + } + mock_instance.return_value = fake_orch + + # requests.request must return a real Response with bytes content + r = Response() + r.status_code = 200 + r._content = b'{"status":"success","data":{}}' + mock_request.return_value = r + self._get('/api/prometheus/rules') - mock_request.assert_called_with('GET', - self.prometheus_host_api + '/rules', - json=None, params={}, - verify=True, cert=None, auth=None) - assert mock_mon_command.called + mock_request.assert_called_with( + 'GET', + self.prometheus_host_api + '/rules', + json=None, + params={}, + verify=True, + cert=None, + auth=None) + self.assertStatus(200) @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", return_value='cephadm') @patch("dashboard.controllers.prometheus.mgr.mon_command", return_value=(1, {}, None)) -- 2.39.5