node-proxy: subclass Thread class

The idea is to subclass Thread so I can catch
exceptions in threads from the main process.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
(cherry picked from commit c04a88c4d24d83a5d1a60341514a98c87fe6f833)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 2fa65f324454e227104a706c8c8295f64aad1edb..58f2901ed7c94a1c78fde299c7af09bf70110ac6 100755 (executable)
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -40,7 +40,7 @@ from threading import Thread, Event
 from urllib.error import HTTPError, URLError
 from urllib.request import urlopen, Request
 from pathlib import Path
-import cephadmlib.node_proxy.server
+from cephadmlib.node_proxy.server import NodeProxy
 
 FuncT = TypeVar('FuncT', bound=Callable)
 
@@ -4915,27 +4915,32 @@ WantedBy=ceph-{fsid}.target
             raise
         return response_str
 
-    def run(self) -> None:
-        self.pull_conf_settings()
-        ssl_ctx = ssl.create_default_context()
-        ssl_ctx.check_hostname = True
-        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
-        ssl_ctx.load_verify_locations(self.ca_path)
+    def init_node_proxy(self, ssl_ctx: Any) -> None:
         node_proxy_data = json.dumps({'keyring': self.keyring,
-                                      'host': self.host})
+                                          'host': self.host})
         node_proxy_data = node_proxy_data.encode('ascii')
         result = self.query_endpoint(data=node_proxy_data,
                                      endpoint='/node-proxy/idrac',
                                      ssl_ctx=ssl_ctx)
         result_json = json.loads(result)
-        t_node_proxy = Thread(target=cephadmlib.node_proxy.server.main,
-                              kwargs={'host': result_json['result']['addr'],
-                                      'username': result_json['result']['username'],
-                                      'password': result_json['result']['password'],
-                                      'data': node_proxy_data,
-                                      'mgr_target_ip': self.target_ip,
-                                      'mgr_target_port': self.target_port})
-        t_node_proxy.start()
+        kwargs = {
+            'host': result_json['result']['addr'],
+            'username': result_json['result']['username'],
+            'password': result_json['result']['password'],
+            'data': node_proxy_data,
+            'mgr_target_ip': self.target_ip,
+            'mgr_target_port': self.target_port,
+        }
+
+        self.t_node_proxy = NodeProxy(**kwargs)
+        self.t_node_proxy.start()
+
+    def run(self) -> None:
+        self.pull_conf_settings()
+        ssl_ctx = ssl.create_default_context()
+        ssl_ctx.check_hostname = True
+        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        ssl_ctx.load_verify_locations(self.ca_path)
 
         try:
             for _ in range(1001):
@@ -4957,7 +4962,16 @@ WantedBy=ceph-{fsid}.target
         if not self.volume_gatherer.is_alive():
             self.volume_gatherer.start()
 
+        # initiate node-proxy thread
+        self.init_node_proxy(ssl_ctx)
+
         while not self.stop:
+            try:
+                _mapper = {True: 'Ok', False: 'Critical'}
+                logger.debug(f'node-proxy status: {_mapper[self.t_node_proxy.check_status()]}')
+            except Exception as e:
+                logger.error(f'node-proxy failure: {e.__class__.__name__}: {e}')
+                self.init_node_proxy(ssl_ctx)
             start_time = time.monotonic()
             ack = self.ack
 

diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
index 9f7ed6c1cb6a21d89a9bad661c6b541e8f1f3bbd..25a42e19f2f36e2a8c517dc2ebde4e8d156f9dde 100644 (file)
--- a/src/cephadm/cephadmlib/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -1,8 +1,9 @@
 import cherrypy
+from threading import Thread
 from .redfish_dell import RedfishDell
 from .reporter import Reporter
 from .util import Config, Logger
-from typing import Dict
+from typing import Dict, Any, Optional
 from .basesystem import BaseSystem
 import sys
 import argparse
@@ -187,59 +188,63 @@ class API:
         return 'use /system or /admin endpoints'
 
 
-def main(host: str = '',
-         username: str = '',
-         password: str = '',
-         data: str = '',
-         mgr_target_ip: str = '',
-         mgr_target_port: str = '') -> None:
-    # TODO: add a check and fail if host/username/password/data aren't passed
-
-    # parser = argparse.ArgumentParser(
-    #     prog='node-proxy',
-    # )
-    # parser.add_argument(
-    #     '--config',
-    #     dest='config',
-    #     type=str,
-    #     required=False,
-    #     default='/etc/ceph/node-proxy.yml'
-    # )
-
-    # args = parser.parse_args()
-    config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
-
-    log = Logger(__name__, level=config.__dict__['logging']['level'])
-
-    host = host
-    username = username
-    password = password
-    data = json.loads(data)
-
-    # create the redfish system and the obsever
-    log.logger.info("Server initialization...")
-    try:
-        system = RedfishDell(host=host,
-                            username=username,
-                            password=password,
-                            system_endpoint='/Systems/System.Embedded.1',
-                            config=config)
-    except RuntimeError:
-        log.logger.error(f"Can't initialize the redfish system.")
-
-    reporter_agent = Reporter(system, data, f"https://{mgr_target_ip}:{mgr_target_port}/node-proxy/data")
-    cherrypy.config.update({
-        'node_proxy': config,
-        'server.socket_port': config.__dict__['server']['port']
-    })
-    c = {'/': {
-        'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
-        'request.dispatch': cherrypy.dispatch.MethodDispatcher()
-    }}
-    system.start_update_loop()
-    reporter_agent.run()
-    cherrypy.quickstart(API(system, reporter_agent, config), config=c)
-
-
-if __name__ == '__main__':
-    main()
+class NodeProxy(Thread):
+    def __init__(self, **kw: Dict[str, Any]) -> None:
+        super().__init__()
+        for k, v in kw.items():
+            setattr(self, k, v)
+        self.exc: Optional[Exception] = None
+
+    def run(self) -> None:
+        try:
+            self.main()
+        except Exception as e:
+            self.exc = e
+            return
+
+    def check_status(self) -> bool:
+        if self.__dict__.get('system') and not self.system.run:
+            raise RuntimeError("node-proxy encountered an error.")
+        if self.exc:
+            raise self.exc
+        return True
+
+    def main(self) -> None:
+        # TODO: add a check and fail if host/username/password/data aren't passed
+
+        config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
+
+        log = Logger(__name__, level=config.__dict__['logging']['level'])
+
+        self.data = json.loads(self.__dict__['data'])
+
+        # create the redfish system and the obsever
+        log.logger.info(f"Server initialization...")
+        try:
+            self.system = RedfishDell(host=self.__dict__['host'],
+                                      username=self.__dict__['username'],
+                                      password=self.__dict__['password'],
+                                      system_endpoint='/Systems/System.Embedded.1',
+                                      config=config)
+        except RuntimeError:
+            log.logger.error("Can't initialize the redfish system.")
+            raise
+
+        try:
+            reporter_agent = Reporter(self.system,
+                                      self.data,
+                                      f"https://{self.__dict__['mgr_target_ip']}:{self.__dict__['mgr_target_port']}/node-proxy/data")
+        except RuntimeError:
+            log.logger.error("Can't initialize the reporter.")
+            raise
+
+        cherrypy.config.update({
+            'node_proxy': config,
+            'server.socket_port': config.__dict__['server']['port']
+        })
+        c = {'/': {
+            'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
+            'request.dispatch': cherrypy.dispatch.MethodDispatcher()
+        }}
+        reporter_agent.run()
+        cherrypy.quickstart(API(self.system, reporter_agent, config), config=c)