From 1f06e876ae72c58ef8165e35ad1f14cb4ecd7132 Mon Sep 17 00:00:00 2001 From: Aashish Sharma Date: Mon, 27 May 2024 11:15:15 +0530 Subject: [PATCH] mgr/dashboard: add api for rgw multisite replication wizard Fixes: https://tracker.ceph.com/issues/66227 Signed-off-by: Aashish Sharma --- .../dashboard/controllers/multi_cluster.py | 3 +- src/pybind/mgr/dashboard/controllers/rgw.py | 26 +- src/pybind/mgr/dashboard/module.py | 5 +- src/pybind/mgr/dashboard/openapi.yaml | 1 - .../mgr/dashboard/services/orchestrator.py | 6 +- .../mgr/dashboard/services/rgw_client.py | 262 ++++++++++++------ src/pybind/mgr/dashboard/services/service.py | 180 ++++++++++++ .../mgr/dashboard/tests/test_rgw_client.py | 4 +- 8 files changed, 392 insertions(+), 95 deletions(-) create mode 100644 src/pybind/mgr/dashboard/services/service.py diff --git a/src/pybind/mgr/dashboard/controllers/multi_cluster.py b/src/pybind/mgr/dashboard/controllers/multi_cluster.py index 8fdecf99d4448..f4b915a81bf00 100644 --- a/src/pybind/mgr/dashboard/controllers/multi_cluster.py +++ b/src/pybind/mgr/dashboard/controllers/multi_cluster.py @@ -5,6 +5,7 @@ import json import re import tempfile import time +from typing import Any, Dict from urllib.parse import urlparse import requests @@ -230,7 +231,7 @@ class MultiCluster(RESTController): @Endpoint('PUT') @UpdatePermission - def set_config(self, config: object): + def set_config(self, config: Dict[str, Any]): multicluster_config = self.load_multi_cluster_config() multicluster_config.update({'current_url': config['url']}) multicluster_config.update({'current_user': config['user']}) diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py index 785b6309fbf70..753325bc49667 100644 --- a/src/pybind/mgr/dashboard/controllers/rgw.py +++ b/src/pybind/mgr/dashboard/controllers/rgw.py @@ -15,6 +15,7 @@ from ..security import Permission, Scope from ..services.auth import AuthManager, JwtManager from ..services.ceph_service import CephService from ..services.rgw_client import _SYNC_GROUP_ID, NoRgwDaemonsException, RgwClient, RgwMultisite +from ..services.service import RgwServiceManager from ..tools import json_str_to_object, str_to_bool from . import APIDoc, APIRouter, BaseController, CreatePermission, \ CRUDCollectionMethod, CRUDEndpoint, DeletePermission, Endpoint, \ @@ -112,6 +113,27 @@ class RgwMultisiteStatus(RESTController): secret_key) return result + @RESTController.Collection(method='POST', path='/multisite-replications') + @allow_empty_body + # pylint: disable=W0102,W0613 + def setup_multisite_replication(self, daemon_name=None, realm_name=None, zonegroup_name=None, + zonegroup_endpoints=None, zone_name=None, zone_endpoints=None, + username=None, cluster_fsid=None): + multisite_instance = RgwMultisite() + result = multisite_instance.setup_multisite_replication(realm_name, zonegroup_name, + zonegroup_endpoints, zone_name, + zone_endpoints, username, + cluster_fsid) + return result + + @RESTController.Collection(method='PUT', path='/setup-rgw-credentials') + @allow_empty_body + # pylint: disable=W0102,W0613 + def restart_rgw_daemons_and_set_credentials(self): + rgw_service_manager_instance = RgwServiceManager() + result = rgw_service_manager_instance.restart_rgw_daemons_and_set_credentials() + return result + @APIRouter('rgw/multisite', Scope.RGW) @APIDoc("RGW Multisite Management API", "RgwMultisite") @@ -1044,11 +1066,9 @@ class RgwRealm(RESTController): @UpdatePermission @allow_empty_body # pylint: disable=W0613 - def import_realm_token(self, realm_token, zone_name, port, placement_spec): + def import_realm_token(self, realm_token, zone_name, port, placement_spec=None): try: - multisite_instance = RgwMultisite() result = CephService.import_realm_token(realm_token, zone_name, port, placement_spec) - multisite_instance.update_period() return result except NoRgwDaemonsException as e: raise DashboardException(e, http_status_code=404, component='rgw') diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py index 2149537c157f0..341a4f00f1be0 100644 --- a/src/pybind/mgr/dashboard/module.py +++ b/src/pybind/mgr/dashboard/module.py @@ -32,7 +32,7 @@ from .grafana import push_local_dashboards from .services import nvmeof_cli # noqa # pylint: disable=unused-import from .services.auth import AuthManager, AuthManagerTool, JwtManager from .services.exception import dashboard_exception_handler -from .services.rgw_client import configure_rgw_credentials +from .services.service import RgwServiceManager from .services.sso import SSO_COMMANDS, handle_sso_command from .settings import handle_option_command, options_command_list, options_schema_list from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \ @@ -417,7 +417,8 @@ class Module(MgrModule, CherryPyConfig): @CLIWriteCommand("dashboard set-rgw-credentials") def set_rgw_credentials(self): try: - configure_rgw_credentials() + rgw_service_manager = RgwServiceManager() + rgw_service_manager.configure_rgw_credentials() except Exception as error: return -errno.EINVAL, '', str(error) diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml index a72370c54538e..ae47035915693 100644 --- a/src/pybind/mgr/dashboard/openapi.yaml +++ b/src/pybind/mgr/dashboard/openapi.yaml @@ -11662,7 +11662,6 @@ paths: - realm_token - zone_name - port - - placement_spec type: object responses: '201': diff --git a/src/pybind/mgr/dashboard/services/orchestrator.py b/src/pybind/mgr/dashboard/services/orchestrator.py index 97776dec335c3..2feeaecc5c79a 100644 --- a/src/pybind/mgr/dashboard/services/orchestrator.py +++ b/src/pybind/mgr/dashboard/services/orchestrator.py @@ -130,11 +130,9 @@ class ServiceManager(ResourceManager): service_ids = [service_ids] completion_list = [ - self.api.service_action('reload', service_type, service_name, - service_id) - for service_name, service_id in service_ids + self.api.service_action('restart', f'{service_type}.{service_id}') + for service_id in service_ids ] - self.api.orchestrator_wait(completion_list) for c in completion_list: raise_if_exception(c) diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py index 17fa33f750851..d0953c2456e9a 100644 --- a/src/pybind/mgr/dashboard/services/rgw_client.py +++ b/src/pybind/mgr/dashboard/services/rgw_client.py @@ -2,25 +2,31 @@ # pylint: disable=C0302 # pylint: disable=too-many-branches # pylint: disable=too-many-lines - +import ast import ipaddress import json import logging import os import re +import time import xml.etree.ElementTree as ET # noqa: N814 from enum import Enum from subprocess import SubprocessError +from urllib.parse import urlparse +import requests from mgr_util import build_url, name_to_config_section from .. import mgr from ..awsauth import S3Auth +from ..controllers.multi_cluster import MultiCluster from ..exceptions import DashboardException from ..rest_client import RequestException, RestClient from ..settings import Settings from ..tools import dict_contains_path, dict_get, json_str_to_object, str_to_bool from .ceph_service import CephService +from .orchestrator import OrchClient +from .service import RgwServiceManager try: from typing import Any, Dict, List, Optional, Tuple, Union @@ -39,14 +45,6 @@ class NoRgwDaemonsException(Exception): super().__init__('No RGW service is running.') -class NoCredentialsException(Exception): - def __init__(self): - super(NoCredentialsException, self).__init__( - 'No RGW credentials found, ' - 'please consult the documentation on how to enable RGW for ' - 'the dashboard.') - - class RgwAdminException(Exception): pass @@ -216,78 +214,6 @@ def _parse_frontend_config(config) -> Tuple[int, bool]: raise LookupError('Failed to determine RGW port from "{}"'.format(config)) -def _parse_secrets(user: str, data: dict) -> Tuple[str, str]: - for key in data.get('keys', []): - if key.get('user') == user and data.get('system') in ['true', True]: - access_key = key.get('access_key') - secret_key = key.get('secret_key') - return access_key, secret_key - return '', '' - - -def _get_user_keys(user: str, realm: Optional[str] = None) -> Tuple[str, str]: - access_key = '' - secret_key = '' - rgw_user_info_cmd = ['user', 'info', '--uid', user] - cmd_realm_option = ['--rgw-realm', realm] if realm else [] - if realm: - rgw_user_info_cmd += cmd_realm_option - try: - _, out, err = mgr.send_rgwadmin_command(rgw_user_info_cmd) - if out: - access_key, secret_key = _parse_secrets(user, out) - if not access_key: - rgw_create_user_cmd = [ - 'user', 'create', - '--uid', user, - '--display-name', 'Ceph Dashboard', - '--system', - ] + cmd_realm_option - _, out, err = mgr.send_rgwadmin_command(rgw_create_user_cmd) - if out: - access_key, secret_key = _parse_secrets(user, out) - if not access_key: - logger.error('Unable to create rgw user "%s": %s', user, err) - except SubprocessError as error: - logger.exception(error) - - return access_key, secret_key - - -def configure_rgw_credentials(): - logger.info('Configuring dashboard RGW credentials') - user = 'dashboard' - realms = [] - access_key = '' - secret_key = '' - try: - _, out, err = mgr.send_rgwadmin_command(['realm', 'list']) - if out: - realms = out.get('realms', []) - if err: - logger.error('Unable to list RGW realms: %s', err) - if realms: - realm_access_keys = {} - realm_secret_keys = {} - for realm in realms: - realm_access_key, realm_secret_key = _get_user_keys(user, realm) - if realm_access_key: - realm_access_keys[realm] = realm_access_key - realm_secret_keys[realm] = realm_secret_key - if realm_access_keys: - access_key = json.dumps(realm_access_keys) - secret_key = json.dumps(realm_secret_keys) - else: - access_key, secret_key = _get_user_keys(user) - - assert access_key and secret_key - Settings.RGW_API_ACCESS_KEY = access_key - Settings.RGW_API_SECRET_KEY = secret_key - except (AssertionError, SubprocessError) as error: - logger.exception(error) - raise NoCredentialsException - - # pylint: disable=R0904 class RgwClient(RestClient): _host = None @@ -348,7 +274,8 @@ class RgwClient(RestClient): # The API access key and secret key are mandatory for a minimal configuration. if not (Settings.RGW_API_ACCESS_KEY and Settings.RGW_API_SECRET_KEY): - configure_rgw_credentials() + rgw_service_manager = RgwServiceManager() + rgw_service_manager.configure_rgw_credentials() daemon_keys = RgwClient._daemons.keys() if not daemon_name: @@ -1142,6 +1069,177 @@ class RgwMultisite: except SubprocessError as error: raise DashboardException(error, http_status_code=500, component='rgw') + def replace_hostname(self, endpoint, hostname_to_ip): + # Replace the hostname in the endpoint URL with its corresponding IP address. + parsed_url = urlparse(endpoint) + hostname = parsed_url.hostname + if hostname in hostname_to_ip: + return endpoint.replace(hostname, hostname_to_ip[hostname]) + return endpoint + + def setup_multisite_replication(self, realm_name: str, zonegroup_name: str, + zonegroup_endpoints: str, zone_name: str, + zone_endpoints: str, username: str, + cluster_fsid: Optional[str] = None): + + # Set up multisite replication for Ceph RGW. + logger.info("Starting multisite replication setup") + orch = OrchClient.instance() + + def get_updated_endpoints(endpoints): + # Update endpoint URLs by replacing hostnames with IP addresses. + logger.debug("Updating endpoints: %s", endpoints) + try: + hostname_to_ip = {host['hostname']: host['addr'] for host in (h.to_json() for h in orch.hosts.list())} # noqa E501 # pylint: disable=line-too-long + updated_endpoints = [self.replace_hostname(endpoint, hostname_to_ip) for endpoint in endpoints.split(',')] # noqa E501 # pylint: disable=line-too-long + logger.debug("Updated endpoints: %s", updated_endpoints) + return updated_endpoints + except Exception as e: + logger.error("Failed to update endpoints: %s", e) + raise + + zonegroup_ip_url = ','.join(get_updated_endpoints(zonegroup_endpoints)) + zone_ip_url = ','.join(get_updated_endpoints(zone_endpoints)) + try: + # Create the realm and zonegroup + logger.info("Creating realm: %s", realm_name) + self.create_realm(realm_name=realm_name, default=True) + logger.info("Creating zonegroup: %s", zonegroup_name) + self.create_zonegroup(realm_name=realm_name, zonegroup_name=zonegroup_name, + default=True, master=True, endpoints=zonegroup_ip_url) + except Exception as e: + logger.error("Failed to create realm or zonegroup: %s", e) + raise + try: + # Create the zone and system user, then modify the zone with user credentials + logger.info("Creating zone: %s", zone_name) + if self.create_zone(zone_name=zone_name, zonegroup_name=zonegroup_name, + default=True, master=True, endpoints=zone_ip_url, + access_key=None, secret_key=None): + logger.info("Creating system user: %s", username) + user_details = self.create_system_user(username, zone_name) + if user_details: + keys = user_details['keys'][0] + logger.info("Modifying zone with user credentials: %s", username) + self.modify_zone(zone_name=zone_name, zonegroup_name=zonegroup_name, + default='true', master='true', endpoints=zone_ip_url, + access_key=keys['access_key'], + secret_key=keys['secret_key']) + except Exception as e: + logger.error("Failed to create zone or system user: %s", e) + raise + try: + # Restart RGW daemons and set credentials + logger.info("Restarting RGW daemons and setting credentials") + rgw_service_manager = RgwServiceManager() + rgw_service_manager.restart_rgw_daemons_and_set_credentials() + except Exception as e: + logger.error("Failed to restart RGW daemons: %s", e) + raise + try: + # Get realm tokens and import to another cluster if specified + logger.info("Getting realm tokens") + realm_token_info = CephService.get_realm_tokens() + + if cluster_fsid and realm_token_info: + logger.info("Importing realm token to cluster: %s", cluster_fsid) + self.import_realm_token_to_cluster(cluster_fsid, realm_name, + realm_token_info, username) + except Exception as e: + logger.error("Failed to get realm tokens or import to cluster: %s", e) + raise + logger.info("Multisite replication setup completed") + return realm_token_info + + def import_realm_token_to_cluster(self, cluster_fsid, realm_name, realm_token_info, username): + logger.info("Importing realm token to cluster: %s", cluster_fsid) + try: + for realm_token in realm_token_info: + if realm_token['realm'] == realm_name: + realm_export_token = realm_token['token'] + break + else: + raise ValueError(f"Realm {realm_name} not found in realm tokens") + multi_cluster_config_str = str(mgr.get_module_option_ex('dashboard', 'MULTICLUSTER_CONFIG')) # noqa E501 # pylint: disable=line-too-long + multi_cluster_config = ast.literal_eval(multi_cluster_config_str) + for fsid, clusters in multi_cluster_config['config'].items(): + if fsid == cluster_fsid: + for cluster_info in clusters: + cluster_token = cluster_info.get('token') + cluster_url = cluster_info.get('url') + break + else: + raise ValueError(f"No cluster token found for fsid: {cluster_fsid}") + break + else: + raise ValueError(f"Cluster fsid {cluster_fsid} not found in multi-cluster config") + if cluster_token: + placement_spec: Dict[str, Dict] = {"placement": {}} + payload = { + 'realm_token': realm_export_token, + 'zone_name': 'new_replicated_zone', + 'port': 81, + 'placement_spec': placement_spec + } + + if not cluster_url.endswith('/'): + cluster_url += '/' + + path = 'api/rgw/realm/import_realm_token' + try: + multi_cluster_instance = MultiCluster() + # pylint: disable=protected-access + response = multi_cluster_instance._proxy(method='POST', base_url=cluster_url, + path=path, payload=payload, + token=cluster_token) + logger.info("Successfully imported realm token to cluster: %s", cluster_fsid) + self.check_user_in_second_cluster(cluster_url, cluster_token, username) + return response + except requests.RequestException as e: + logger.error("Could not reach %s: %s", cluster_url, e) + raise DashboardException(f"Could not reach {cluster_url}: {e}", + http_status_code=404, component='dashboard') + except json.JSONDecodeError as e: + logger.error("Error parsing Dashboard API response: %s", e.msg) + raise DashboardException(f"Error parsing Dashboard API response: {e.msg}", + component='dashboard') + except Exception as e: + logger.error("Failed to import realm token to cluster: %s", e) + raise + + def check_user_in_second_cluster(self, cluster_url, cluster_token, username): + logger.info("Checking for user %s in the second cluster", username) + path = 'api/rgw/zone/get_user_list?zoneName=new_replicated_zone' + user_found = False + start_time = time.time() + while not user_found: + if time.time() - start_time > 120: # Timeout after 2 minutes + logger.error("Timeout reached while waiting for user %s to appear \ + in the second cluster", username) + raise DashboardException(code='user_replication_timeout', + msg="Timeout reached while waiting for \ + user %s to appear in the second cluster." % username) + try: + multi_cluster_instance = MultiCluster() + # pylint: disable=protected-access + user_content = multi_cluster_instance._proxy(method='GET', base_url=cluster_url, + path=path, token=cluster_token) + logger.info("User content in the second cluster: %s", user_content) + for user in user_content: + if user['user_id'] == username: + user_found = True + logger.info("User %s found in the second cluster", username) + # pylint: disable=protected-access + restart_daemons_content = multi_cluster_instance._proxy(method='PUT', base_url=cluster_url, # noqa E501 # pylint: disable=line-too-long + path='ui-api/rgw/multisite/setup-rgw-credentials', # noqa E501 # pylint: disable=line-too-long + token=cluster_token) # noqa E501 # pylint: disable=line-too-long + logger.info("Restarted RGW daemons in the second cluster: %s", restart_daemons_content) # noqa E501 # pylint: disable=line-too-long + break + except requests.RequestException as e: + logger.error("Error checking user in the second cluster: %s", e) + logger.info("User %s not found yet, retrying in 5 seconds", username) + time.sleep(5) + def create_realm(self, realm_name: str, default: bool): rgw_realm_create_cmd = ['realm', 'create'] cmd_create_realm_options = ['--rgw-realm', realm_name] diff --git a/src/pybind/mgr/dashboard/services/service.py b/src/pybind/mgr/dashboard/services/service.py new file mode 100644 index 0000000000000..792604dcc591d --- /dev/null +++ b/src/pybind/mgr/dashboard/services/service.py @@ -0,0 +1,180 @@ +import json +import logging +import time +from subprocess import SubprocessError + +try: + from typing import Optional, Tuple +except ImportError: + pass # For typing only + +from .. import mgr +from ..exceptions import DashboardException +from ..settings import Settings +from .orchestrator import OrchClient + +logger = logging.getLogger('service') + + +class NoCredentialsException(Exception): + def __init__(self): + super(NoCredentialsException, self).__init__( + 'No RGW credentials found, ' + 'please consult the documentation on how to enable RGW for ' + 'the dashboard.') + + +def verify_service_restart(service_type: str, service_id: str): + orch = OrchClient.instance() + service_name = f'{service_type}.{service_id}' + + logger.info("Getting initial service info for: %s", service_name) + info = orch.services.get(service_name)[0].to_dict() + last_refreshed = info['status']['last_refresh'] + + logger.info("Reloading service: %s", service_name) + orch.services.reload(service_type, service_id) + + logger.info("Waiting for service refresh: %s", service_name) + wait_for_refresh(orch, service_name, last_refreshed) + + logger.info("Checking daemon status for: %s", service_name) + daemon_status = wait_for_daemon_to_start(orch, service_name) + return daemon_status + + +def wait_for_refresh(orch, service_name, last_refreshed): + orch = OrchClient.instance() + logger.info("Waiting for service %s to refresh", service_name) + + while True: + updated_info = orch.services.get(service_name)[0].to_dict() + if updated_info['status']['last_refresh'] != last_refreshed: + logger.info("Service %s refreshed", service_name) + break + + +def wait_for_daemon_to_start(orch, service_name): + orch = OrchClient.instance() + start_time = time.time() + logger.info("Waiting for daemon %s to start", service_name) + + while True: + daemons = [d.to_dict() for d in orch.services.list_daemons(service_name=service_name)] + all_running = True + + for daemon in daemons: + daemon_state = daemon['status_desc'] + logger.debug("Daemon %s state: %s", daemon['daemon_id'], daemon_state) + + if daemon_state in ('unknown', 'error', 'stopped'): + logger.error("Failed to restart daemon %s for service %s. State is %s", daemon['daemon_id'], service_name, daemon_state) # noqa E501 # pylint: disable=line-too-long + raise DashboardException( + code='daemon_restart_failed', + msg="Failed to restart the daemon %s. Daemon state is %s." % (service_name, daemon_state) # noqa E501 # pylint: disable=line-too-long + ) + if daemon_state != 'running': + all_running = False + + if all_running: + logger.info("All daemons for service %s are running", service_name) + return True + + if time.time() - start_time > 10: + logger.error("Timeout reached while waiting for daemon %s to start", service_name) + raise DashboardException( + code='daemon_restart_timeout', + msg="Timeout reached while waiting for daemon %s to start." % service_name + ) + return False + + +class RgwServiceManager: + def restart_rgw_daemons_and_set_credentials(self): + # Restart RGW daemons and set credentials. + logger.info("Restarting RGW daemons and setting credentials") + orch = OrchClient.instance() + services, _ = orch.services.list(service_type='rgw', offset=0) + + all_daemons_up = True + for service in services: + logger.info("Verifying service restart for: %s", service['service_id']) + daemons_up = verify_service_restart('rgw', service['service_id']) + if not daemons_up: + logger.error("Service %s restart verification failed", service['service_id']) + all_daemons_up = False + + if all_daemons_up: + logger.info("All daemons are up, configuring RGW credentials") + self.configure_rgw_credentials() + else: + logger.error("Not all daemons are up, skipping RGW credentials configuration") + + def _parse_secrets(self, user: str, data: dict) -> Tuple[str, str]: + for key in data.get('keys', []): + if key.get('user') == user and data.get('system') in ['true', True]: + access_key = key.get('access_key') + secret_key = key.get('secret_key') + return access_key, secret_key + return '', '' + + def _get_user_keys(self, user: str, realm: Optional[str] = None) -> Tuple[str, str]: + access_key = '' + secret_key = '' + rgw_user_info_cmd = ['user', 'info', '--uid', user] + cmd_realm_option = ['--rgw-realm', realm] if realm else [] + if realm: + rgw_user_info_cmd += cmd_realm_option + try: + _, out, err = mgr.send_rgwadmin_command(rgw_user_info_cmd) + if out: + access_key, secret_key = self._parse_secrets(user, out) + if not access_key: + rgw_create_user_cmd = [ + 'user', 'create', + '--uid', user, + '--display-name', 'Ceph Dashboard', + '--system', + ] + cmd_realm_option + _, out, err = mgr.send_rgwadmin_command(rgw_create_user_cmd) + if out: + access_key, secret_key = self._parse_secrets(user, out) + if not access_key: + logger.error('Unable to create rgw user "%s": %s', user, err) + except SubprocessError as error: + logger.exception(error) + + return access_key, secret_key + + def configure_rgw_credentials(self): + logger.info('Configuring dashboard RGW credentials') + user = 'dashboard' + realms = [] + access_key = '' + secret_key = '' + try: + _, out, err = mgr.send_rgwadmin_command(['realm', 'list']) + if out: + realms = out.get('realms', []) + if err: + logger.error('Unable to list RGW realms: %s', err) + if realms: + realm_access_keys = {} + realm_secret_keys = {} + for realm in realms: + realm_access_key, realm_secret_key = self._get_user_keys(user, realm) + if realm_access_key: + realm_access_keys[realm] = realm_access_key + realm_secret_keys[realm] = realm_secret_key + if realm_access_keys: + access_key = json.dumps(realm_access_keys) + secret_key = json.dumps(realm_secret_keys) + else: + access_key, secret_key = self._get_user_keys(user) + + assert access_key and secret_key + Settings.RGW_API_ACCESS_KEY = access_key + Settings.RGW_API_SECRET_KEY = secret_key + except (AssertionError, SubprocessError) as error: + logger.exception(error) + raise NoCredentialsException diff --git a/src/pybind/mgr/dashboard/tests/test_rgw_client.py b/src/pybind/mgr/dashboard/tests/test_rgw_client.py index 4949ba36bf211..1618a522867c0 100644 --- a/src/pybind/mgr/dashboard/tests/test_rgw_client.py +++ b/src/pybind/mgr/dashboard/tests/test_rgw_client.py @@ -6,8 +6,8 @@ from unittest.mock import Mock, patch from .. import mgr from ..exceptions import DashboardException -from ..services.rgw_client import NoCredentialsException, \ - NoRgwDaemonsException, RgwClient, _parse_frontend_config +from ..services.rgw_client import NoRgwDaemonsException, RgwClient, _parse_frontend_config +from ..services.service import NoCredentialsException from ..settings import Settings from ..tests import CLICommandTestMixin, RgwStub -- 2.39.5