From 7c6ea18d6e0bcaad73d3fb8a2138f4b31894b138 Mon Sep 17 00:00:00 2001 From: yaelazulay-redhat Date: Mon, 16 Mar 2026 22:37:22 +0200 Subject: [PATCH] mgr/dashboard: use cephadm root CA for RGW SSL and improve error handling Problem: Dashboard fails to access object pages when RGW is deployed with SSL using cephadm-signed certificates. Root cause: RGW REST API connection fails with SSL certificate verification error because the cephadm root CA certificate that signed the RGW SSL certificates is not in the dashboard's trust store. Code Fixes: 1. rgw_client.py: Added _get_ssl_ca_bundle() which fetches the cephadm root CA certificate from the cert store and writes it atomically (via a temp file and os.replace) to a fixed path (/tmp/ceph-dashboard-ca/rgw-cephadm-root-ca.pem), returning the file path for SSL verification. Notes: - The file is written once per mgr process lifetime and reused by all RgwClient instances. On mgr restart it is refetched and overwritten. - A dedicated subdirectory (/tmp/ceph-dashboard-ca/) is used because /tmp has the sticky bit set, which prevents os.replace from overwriting files owned by a different user. 2. rest_client.py Fixed secondary that handle_connection_error crash - when the initial SSL error occurred, the error handler itself crashed trying to process the exception, because it assumed reason.args[0] was always a string, but for SSL errors it's an SSLError object. Fixes: https://tracker.ceph.com/issues/74393 Signed-off-by: Yael Azulay --- src/pybind/mgr/dashboard/rest_client.py | 43 +++++++------ .../mgr/dashboard/services/rgw_client.py | 60 ++++++++++++++++++- 2 files changed, 85 insertions(+), 18 deletions(-) diff --git a/src/pybind/mgr/dashboard/rest_client.py b/src/pybind/mgr/dashboard/rest_client.py index 9e27cb57802d..230f4871e18f 100644 --- a/src/pybind/mgr/dashboard/rest_client.py +++ b/src/pybind/mgr/dashboard/rest_client.py @@ -471,18 +471,27 @@ class RestClient(object): return resp def handle_connection_error(self, exception, method): + errno = "n/a" + strerror = "n/a" if exception.args: if isinstance(exception.args[0], SSLError): - errno = "n/a" - strerror = "SSL error. Probably trying to access a non " \ - "SSL connection." - logger.error("%s REST API failed %s, SSL error (url=%s).", - self.client_name, method.upper(), exception.request.url) + strerror = str(exception.args[0]).strip() or ( + "SSL error. Probably trying to access a non SSL connection.") + logger.error("%s REST API failed %s, SSL error (url=%s): %s", + self.client_name, method.upper(), exception.request.url, + strerror) else: try: - match = re.match(r'.*: \[Errno (-?\d+)\] (.+)', - exception.args[0].reason.args[0]) - except AttributeError: + reason_str = exception.args[0].reason.args[0] + if reason_str is not None: + reason_str = str(reason_str) + else: + reason_str = '' + except (AttributeError, TypeError, IndexError): + reason_str = str(exception.args[0]) if exception.args else '' + try: + match = re.match(r'.*: \[Errno (-?\d+)\] (.+)', reason_str) + except TypeError: match = None if match: errno = match.group(1) @@ -492,23 +501,23 @@ class RestClient(object): "[errno: %s] %s", self.client_name, method.upper(), exception.request.url, errno, strerror) - else: - errno = "n/a" - strerror = "n/a" - logger.error( - "%s REST API failed %s, connection error (url=%s).", - self.client_name, method.upper(), exception.request.url) - else: - errno = "n/a" - strerror = "n/a" + + if errno == "n/a" and strerror == "n/a": logger.error("%s REST API failed %s, connection error (url=%s).", self.client_name, method.upper(), exception.request.url) + if errno != "n/a": exception_msg = ( "{} REST API cannot be reached: {} [errno {}]. " "Please check your configuration and that the API endpoint" " is accessible" .format(self.client_name, strerror, errno)) + elif strerror != "n/a": + exception_msg = ( + "{} REST API cannot be reached: {}. " + "Please check your configuration and that the API endpoint" + " is accessible" + .format(self.client_name, strerror)) else: exception_msg = ( "{} REST API cannot be reached. Please check " diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py index 5355a9ef1856..2b5634fb11bd 100755 --- a/src/pybind/mgr/dashboard/services/rgw_client.py +++ b/src/pybind/mgr/dashboard/services/rgw_client.py @@ -8,6 +8,7 @@ import json import logging import os import re +import tempfile import time import uuid import xml.etree.ElementTree as ET # noqa: N814 @@ -30,6 +31,7 @@ from .. import mgr from ..awsauth import S3Auth from ..controllers.multi_cluster import MultiCluster from ..exceptions import DashboardException +from ..model.certificate import CEPHADM_ROOT_CA_CERT from ..rest_client import RequestException, RestClient from ..settings import Settings from ..tools import dict_contains_path, dict_get, json_str_to_object, str_to_bool @@ -79,6 +81,10 @@ class RgwDaemon: zone_name: str +_RGW_CEPHADM_CA_BUNDLE_DIR = os.path.join(tempfile.gettempdir(), 'ceph-dashboard-ca') +_RGW_CEPHADM_CA_BUNDLE_PATH = os.path.join(_RGW_CEPHADM_CA_BUNDLE_DIR, 'rgw-cephadm-root-ca.pem') + + def _get_daemons() -> Dict[str, RgwDaemon]: """ Retrieve RGW daemon info from MGR. @@ -244,6 +250,7 @@ class RgwClient(RestClient): _config_instances = {} # type: Dict[str, RgwClient] _rgw_settings_snapshot = None _daemons: Dict[str, RgwDaemon] = {} + _ssl_ca_bundle_written: bool = False daemon: RgwDaemon got_keys_from_config: bool userid: str @@ -386,6 +393,55 @@ class RgwClient(RestClient): self.auth = S3Auth(keys['access_key'], keys['secret_key'], service_url=self.service_url) + @classmethod + def _get_ssl_ca_bundle(cls, fallback): + """Fetch the cephadm root CA cert and write it to a fixed file for SSL verification. + + The file lives under a mgr-owned subdirectory of /tmp so that + os.replace works without sticky-bit issues. Falls back to the provided + default if the cert store is unavailable or the root CA certificate is + not found. The file is refreshed after _SSL_CA_BUNDLE_TTL seconds to + pick up CA rotations. + """ + path = _RGW_CEPHADM_CA_BUNDLE_PATH + if (cls._ssl_ca_bundle_written + and os.path.isfile(path) + and os.path.getsize(path) > 0): + return path + try: + orch = OrchClient.instance() + if not orch.available(): + logger.warning("Orchestrator is not available, " + "cannot fetch root CA cert for RGW SSL verification") + return fallback + root_ca_cert = orch.cert_store.get_cert(CEPHADM_ROOT_CA_CERT) + if not root_ca_cert: + logger.warning("cephadm root CA cert not found in cert store, " + "falling back to default SSL verification") + return fallback + os.makedirs(_RGW_CEPHADM_CA_BUNDLE_DIR, mode=0o700, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + dir=_RGW_CEPHADM_CA_BUNDLE_DIR, prefix='.rgw-ca-', suffix='.tmp') + try: + with os.fdopen(fd, 'wb') as tmp_file: + os.fchmod(tmp_file.fileno(), 0o600) + tmp_file.write(root_ca_cert.encode('utf-8')) + os.replace(tmp_path, path) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + cls._ssl_ca_bundle_written = True + logger.info("Using cephadm root CA cert for RGW SSL verification: %s", path) + return path + except Exception as ex: # pylint: disable=broad-except + logger.warning("Could not fetch cephadm root CA cert: %s. " + "Falling back to default SSL verification", ex, + exc_info=True) + return fallback + def __init__(self, access_key: str, secret_key: str, @@ -398,6 +454,8 @@ class RgwClient(RestClient): http_status_code=404, component='rgw') ssl_verify = Settings.RGW_API_SSL_VERIFY + if ssl_verify and daemon.ssl: + ssl_verify = RgwClient._get_ssl_ca_bundle(ssl_verify) self.admin_path = Settings.RGW_API_ADMIN_RESOURCE self.service_url = build_url(host=daemon.host, port=daemon.port) @@ -422,7 +480,7 @@ class RgwClient(RestClient): component='rgw') self.daemon = daemon - logger.info("Created new connection: daemon=%s, host=%s, port=%s, ssl=%d, sslverify=%d", + logger.info("Created new connection: daemon=%s, host=%s, port=%s, ssl=%d, sslverify=%s", daemon.name, daemon.host, daemon.port, daemon.ssl, ssl_verify) @RestClient.api_get('/', resp_structure='[0] > (ID & DisplayName)') -- 2.47.3