node-proxy: add a retry on redfish_client.get_path() calls

The idea is to retry multiple times before stating the endpoint is
definitely unreachable.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
(cherry picked from commit c8f31a1ef01d777e9ef8aae1a895dfcf0a6dea8b)

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index bfec7d9ad4e9b5a514d4850cc938efdb5a944eab..6a3e384893d2e3a00507f069f3fae4da9afcb08a 100644 (file)
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -47,9 +47,8 @@ class RedFishClient:
             response = self.redfish_obj.get(path)
             return response.dict
         except Exception as e:
-            # TODO
-            log.logger.error(f"Error detected.\n{e}")
-            pass
+            log.logger.error(f"Error getting path:\n{e}")
+            return {}
 
     def logout(self) -> None:
         log.logger.info('logging out...')

diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
index 80e54f1cff3152094472d7a4c55fc14edf09d3f0..f940a9ec3c30b3a04f1c824bbf5f890d73530269 100644 (file)
--- a/src/cephadm/node-proxy/redfish_dell.py
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -14,12 +14,12 @@ class RedfishDell(RedfishSystem):
     def _update_network(self) -> None:
         net_path = self._system['EthernetInterfaces']['@odata.id']
         log.logger.info("Updating network")
-        network_info = self.client.get_path(net_path)
+        network_info = self._get_path(net_path)
         self._system['network'] = {}
         result: Dict[str, Dict[str, Dict]] = dict()
         for interface in network_info['Members']:
             interface_path = interface['@odata.id']
-            interface_info = self.client.get_path(interface_path)
+            interface_info = self._get_path(interface_path)
             interface_id = interface_info['Id']
             result[interface_id] = dict()
             result[interface_id]['description'] = interface_info['Description']
@@ -31,12 +31,12 @@ class RedfishDell(RedfishSystem):
     def _update_processors(self) -> None:
         cpus_path = self._system['Processors']['@odata.id']
         log.logger.info("Updating processors")
-        cpus_info = self.client.get_path(cpus_path)
+        cpus_info = self._get_path(cpus_path)
         self._system['processors'] = {}
         result: Dict[str, Dict[str, Dict]] = dict()
         for cpu in cpus_info['Members']:
             cpu_path = cpu['@odata.id']
-            cpu_info = self.client.get_path(cpu_path)
+            cpu_info = self._get_path(cpu_path)
             cpu_id = cpu_info['Id']
             result[cpu_id] = dict()
             result[cpu_id]['description'] = cpu_info['Description']
@@ -51,14 +51,14 @@ class RedfishDell(RedfishSystem):
     def _update_storage(self) -> None:
         storage_path = self._system['Storage']['@odata.id']
         log.logger.info("Updating storage")
-        storage_info = self.client.get_path(storage_path)
+        storage_info = self._get_path(storage_path)
         result: Dict[str, Dict[str, Dict]] = dict()
         for storage in storage_info['Members']:
             entity_path = storage['@odata.id']
-            entity_info = self.client.get_path(entity_path)
+            entity_info = self._get_path(entity_path)
             for drive in entity_info['Drives']:
                 drive_path = drive['@odata.id']
-                drive_info = self.client.get_path(drive_path)
+                drive_info = self._get_path(drive_path)
                 drive_id = drive_info['Id']
                 result[drive_id] = dict()
                 result[drive_id]['description'] = drive_info['Description']

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 7b95005b9546e99c886c3c7b81cbe7c5f8f9505d..0dc269deb50893302ddde211af0b0ab9cc9ea401 100644 (file)
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -2,7 +2,7 @@ from system import System
 from redfish_client import RedFishClient
 from threading import Thread, Lock
 from time import sleep
-from util import Logger
+from util import Logger, retry
 from typing import Dict, Any
 
 log = Logger(__name__)
@@ -26,6 +26,14 @@ class RedfishSystem(System):
         self.previous_data: Dict = {}
         self.lock: Lock = Lock()
 
+    @retry(retries=10, delay=2)
+    def _get_path(self, path: str) -> Dict:
+        result = self.client.get_path(path)
+        if result is None:
+            log.logger.error(f"The client reported an error when getting path: {path}")
+            raise RuntimeError(f"Could not get path: {path}")
+        return result
+
     def start_client(self) -> None:
         log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)