mgr/dashboard: use cephadm root CA for RGW SSL and improve error handling

Problem: Dashboard fails to access object pages when RGW is deployed with SSL using cephadm-signed certificates.

Root cause: RGW REST API connection fails with SSL certificate verification error because the cephadm root CA certificate that signed the RGW SSL certificates is not in the dashboard's trust store.

Code Fixes:
1. rgw_client.py:
Added _get_ssl_ca_bundle() which fetches the cephadm root CA certificate from the cert store and writes it atomically (via a temp file and os.replace) to a fixed path (/tmp/ceph-dashboard-ca/rgw-cephadm-root-ca.pem), returning the file path for SSL verification.
Notes:
- The file is written once per mgr process lifetime and reused by all RgwClient instances. On mgr restart it is refetched and overwritten.
- A dedicated subdirectory (/tmp/ceph-dashboard-ca/) is used because /tmp has the sticky bit set, which prevents os.replace from overwriting files owned by a different user.

2. rest_client.py
Fixed secondary that handle_connection_error crash - when the initial SSL error occurred, the error handler itself crashed trying to process the exception, because it assumed reason.args[0] was always a string, but for SSL errors it's an SSLError object.

Fixes: https://tracker.ceph.com/issues/74393
Signed-off-by: Yael Azulay <yazulay@redhat.com>

diff --git a/src/pybind/mgr/dashboard/rest_client.py b/src/pybind/mgr/dashboard/rest_client.py
index 9e27cb57802da677b12701c9e3f731f4643df584..230f4871e18fe2e9c98d28f315313349cb4f4092 100644 (file)
--- a/src/pybind/mgr/dashboard/rest_client.py
+++ b/src/pybind/mgr/dashboard/rest_client.py
@@ -471,18 +471,27 @@ class RestClient(object):
         return resp
 
     def handle_connection_error(self, exception, method):
+        errno = "n/a"
+        strerror = "n/a"
         if exception.args:
             if isinstance(exception.args[0], SSLError):
-                errno = "n/a"
-                strerror = "SSL error. Probably trying to access a non " \
-                           "SSL connection."
-                logger.error("%s REST API failed %s, SSL error (url=%s).",
-                             self.client_name, method.upper(), exception.request.url)
+                strerror = str(exception.args[0]).strip() or (
+                    "SSL error. Probably trying to access a non SSL connection.")
+                logger.error("%s REST API failed %s, SSL error (url=%s): %s",
+                             self.client_name, method.upper(), exception.request.url,
+                             strerror)
             else:
                 try:
-                    match = re.match(r'.*: \[Errno (-?\d+)\] (.+)',
-                                     exception.args[0].reason.args[0])
-                except AttributeError:
+                    reason_str = exception.args[0].reason.args[0]
+                    if reason_str is not None:
+                        reason_str = str(reason_str)
+                    else:
+                        reason_str = ''
+                except (AttributeError, TypeError, IndexError):
+                    reason_str = str(exception.args[0]) if exception.args else ''
+                try:
+                    match = re.match(r'.*: \[Errno (-?\d+)\] (.+)', reason_str)
+                except TypeError:
                     match = None
                 if match:
                     errno = match.group(1)
@@ -492,23 +501,23 @@ class RestClient(object):
                         "[errno: %s] %s",
                         self.client_name, method.upper(), exception.request.url,
                         errno, strerror)
-                else:
-                    errno = "n/a"
-                    strerror = "n/a"
-                    logger.error(
-                        "%s REST API failed %s, connection error (url=%s).",
-                        self.client_name, method.upper(), exception.request.url)
-        else:
-            errno = "n/a"
-            strerror = "n/a"
+
+        if errno == "n/a" and strerror == "n/a":
             logger.error("%s REST API failed %s, connection error (url=%s).",
                          self.client_name, method.upper(), exception.request.url)
+
         if errno != "n/a":
             exception_msg = (
                 "{} REST API cannot be reached: {} [errno {}]. "
                 "Please check your configuration and that the API endpoint"
                 " is accessible"
                 .format(self.client_name, strerror, errno))
+        elif strerror != "n/a":
+            exception_msg = (
+                "{} REST API cannot be reached: {}. "
+                "Please check your configuration and that the API endpoint"
+                " is accessible"
+                .format(self.client_name, strerror))
         else:
             exception_msg = (
                 "{} REST API cannot be reached. Please check "

diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index 5355a9ef18562530c58871c469a826413bf7faac..2b5634fb11bdeba0b925d49a4f657be21b06a19c 100755 (executable)
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -8,6 +8,7 @@ import json
 import logging
 import os
 import re
+import tempfile
 import time
 import uuid
 import xml.etree.ElementTree as ET  # noqa: N814
@@ -30,6 +31,7 @@ from .. import mgr
 from ..awsauth import S3Auth
 from ..controllers.multi_cluster import MultiCluster
 from ..exceptions import DashboardException
+from ..model.certificate import CEPHADM_ROOT_CA_CERT
 from ..rest_client import RequestException, RestClient
 from ..settings import Settings
 from ..tools import dict_contains_path, dict_get, json_str_to_object, str_to_bool
@@ -79,6 +81,10 @@ class RgwDaemon:
     zone_name: str
 
 
+_RGW_CEPHADM_CA_BUNDLE_DIR = os.path.join(tempfile.gettempdir(), 'ceph-dashboard-ca')
+_RGW_CEPHADM_CA_BUNDLE_PATH = os.path.join(_RGW_CEPHADM_CA_BUNDLE_DIR, 'rgw-cephadm-root-ca.pem')
+
+
 def _get_daemons() -> Dict[str, RgwDaemon]:
     """
     Retrieve RGW daemon info from MGR.
@@ -244,6 +250,7 @@ class RgwClient(RestClient):
     _config_instances = {}  # type: Dict[str, RgwClient]
     _rgw_settings_snapshot = None
     _daemons: Dict[str, RgwDaemon] = {}
+    _ssl_ca_bundle_written: bool = False
     daemon: RgwDaemon
     got_keys_from_config: bool
     userid: str
@@ -386,6 +393,55 @@ class RgwClient(RestClient):
         self.auth = S3Auth(keys['access_key'], keys['secret_key'],
                            service_url=self.service_url)
 
+    @classmethod
+    def _get_ssl_ca_bundle(cls, fallback):
+        """Fetch the cephadm root CA cert and write it to a fixed file for SSL verification.
+
+        The file lives under a mgr-owned subdirectory of /tmp so that
+        os.replace works without sticky-bit issues.  Falls back to the provided
+        default if the cert store is unavailable or the root CA certificate is
+        not found.  The file is refreshed after _SSL_CA_BUNDLE_TTL seconds to
+        pick up CA rotations.
+        """
+        path = _RGW_CEPHADM_CA_BUNDLE_PATH
+        if (cls._ssl_ca_bundle_written
+                and os.path.isfile(path)
+                and os.path.getsize(path) > 0):
+            return path
+        try:
+            orch = OrchClient.instance()
+            if not orch.available():
+                logger.warning("Orchestrator is not available, "
+                               "cannot fetch root CA cert for RGW SSL verification")
+                return fallback
+            root_ca_cert = orch.cert_store.get_cert(CEPHADM_ROOT_CA_CERT)
+            if not root_ca_cert:
+                logger.warning("cephadm root CA cert not found in cert store, "
+                               "falling back to default SSL verification")
+                return fallback
+            os.makedirs(_RGW_CEPHADM_CA_BUNDLE_DIR, mode=0o700, exist_ok=True)
+            fd, tmp_path = tempfile.mkstemp(
+                dir=_RGW_CEPHADM_CA_BUNDLE_DIR, prefix='.rgw-ca-', suffix='.tmp')
+            try:
+                with os.fdopen(fd, 'wb') as tmp_file:
+                    os.fchmod(tmp_file.fileno(), 0o600)
+                    tmp_file.write(root_ca_cert.encode('utf-8'))
+                os.replace(tmp_path, path)
+            except Exception:
+                try:
+                    os.unlink(tmp_path)
+                except OSError:
+                    pass
+                raise
+            cls._ssl_ca_bundle_written = True
+            logger.info("Using cephadm root CA cert for RGW SSL verification: %s", path)
+            return path
+        except Exception as ex:  # pylint: disable=broad-except
+            logger.warning("Could not fetch cephadm root CA cert: %s. "
+                           "Falling back to default SSL verification", ex,
+                           exc_info=True)
+        return fallback
+
     def __init__(self,
                  access_key: str,
                  secret_key: str,
@@ -398,6 +454,8 @@ class RgwClient(RestClient):
                                      http_status_code=404,
                                      component='rgw')
         ssl_verify = Settings.RGW_API_SSL_VERIFY
+        if ssl_verify and daemon.ssl:
+            ssl_verify = RgwClient._get_ssl_ca_bundle(ssl_verify)
         self.admin_path = Settings.RGW_API_ADMIN_RESOURCE
         self.service_url = build_url(host=daemon.host, port=daemon.port)
 
@@ -422,7 +480,7 @@ class RgwClient(RestClient):
                                      component='rgw')
         self.daemon = daemon
 
-        logger.info("Created new connection: daemon=%s, host=%s, port=%s, ssl=%d, sslverify=%d",
+        logger.info("Created new connection: daemon=%s, host=%s, port=%s, ssl=%d, sslverify=%s",
                     daemon.name, daemon.host, daemon.port, daemon.ssl, ssl_verify)
 
     @RestClient.api_get('/', resp_structure='[0] > (ID & DisplayName)')