From a74a2a0cdd5383256334fd215bfa8da79806a28c Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Fri, 9 Jun 2023 15:03:24 +0200 Subject: [PATCH] node-proxy: add a retry on redfish_client.get_path() calls The idea is to retry multiple times before stating the endpoint is definitely unreachable. Signed-off-by: Guillaume Abrioux (cherry picked from commit c8f31a1ef01d777e9ef8aae1a895dfcf0a6dea8b) --- src/cephadm/node-proxy/redfish_client.py | 5 ++--- src/cephadm/node-proxy/redfish_dell.py | 14 +++++++------- src/cephadm/node-proxy/redfish_system.py | 10 +++++++++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py index bfec7d9ad4e..6a3e384893d 100644 --- a/src/cephadm/node-proxy/redfish_client.py +++ b/src/cephadm/node-proxy/redfish_client.py @@ -47,9 +47,8 @@ class RedFishClient: response = self.redfish_obj.get(path) return response.dict except Exception as e: - # TODO - log.logger.error(f"Error detected.\n{e}") - pass + log.logger.error(f"Error getting path:\n{e}") + return {} def logout(self) -> None: log.logger.info('logging out...') diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py index 80e54f1cff3..f940a9ec3c3 100644 --- a/src/cephadm/node-proxy/redfish_dell.py +++ b/src/cephadm/node-proxy/redfish_dell.py @@ -14,12 +14,12 @@ class RedfishDell(RedfishSystem): def _update_network(self) -> None: net_path = self._system['EthernetInterfaces']['@odata.id'] log.logger.info("Updating network") - network_info = self.client.get_path(net_path) + network_info = self._get_path(net_path) self._system['network'] = {} result: Dict[str, Dict[str, Dict]] = dict() for interface in network_info['Members']: interface_path = interface['@odata.id'] - interface_info = self.client.get_path(interface_path) + interface_info = self._get_path(interface_path) interface_id = interface_info['Id'] result[interface_id] = dict() result[interface_id]['description'] = interface_info['Description'] @@ -31,12 +31,12 @@ class RedfishDell(RedfishSystem): def _update_processors(self) -> None: cpus_path = self._system['Processors']['@odata.id'] log.logger.info("Updating processors") - cpus_info = self.client.get_path(cpus_path) + cpus_info = self._get_path(cpus_path) self._system['processors'] = {} result: Dict[str, Dict[str, Dict]] = dict() for cpu in cpus_info['Members']: cpu_path = cpu['@odata.id'] - cpu_info = self.client.get_path(cpu_path) + cpu_info = self._get_path(cpu_path) cpu_id = cpu_info['Id'] result[cpu_id] = dict() result[cpu_id]['description'] = cpu_info['Description'] @@ -51,14 +51,14 @@ class RedfishDell(RedfishSystem): def _update_storage(self) -> None: storage_path = self._system['Storage']['@odata.id'] log.logger.info("Updating storage") - storage_info = self.client.get_path(storage_path) + storage_info = self._get_path(storage_path) result: Dict[str, Dict[str, Dict]] = dict() for storage in storage_info['Members']: entity_path = storage['@odata.id'] - entity_info = self.client.get_path(entity_path) + entity_info = self._get_path(entity_path) for drive in entity_info['Drives']: drive_path = drive['@odata.id'] - drive_info = self.client.get_path(drive_path) + drive_info = self._get_path(drive_path) drive_id = drive_info['Id'] result[drive_id] = dict() result[drive_id]['description'] = drive_info['Description'] diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py index 7b95005b954..0dc269deb50 100644 --- a/src/cephadm/node-proxy/redfish_system.py +++ b/src/cephadm/node-proxy/redfish_system.py @@ -2,7 +2,7 @@ from system import System from redfish_client import RedFishClient from threading import Thread, Lock from time import sleep -from util import Logger +from util import Logger, retry from typing import Dict, Any log = Logger(__name__) @@ -26,6 +26,14 @@ class RedfishSystem(System): self.previous_data: Dict = {} self.lock: Lock = Lock() + @retry(retries=10, delay=2) + def _get_path(self, path: str) -> Dict: + result = self.client.get_path(path) + if result is None: + log.logger.error(f"The client reported an error when getting path: {path}") + raise RuntimeError(f"Could not get path: {path}") + return result + def start_client(self) -> None: log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}") self.client = RedFishClient(self.host, self.username, self.password) -- 2.39.5