node-proxy: refactor config loading

This commit renames CONFIG to DEFAULTS and add load_config() with
deep merge, refactor Config to use path + defaults and makes
node-proxy config path configurable via bootstrap JSON or env.

Fixes: https://tracker.ceph.com/issues/74749
Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>

diff --git a/src/ceph-node-proxy/ceph_node_proxy/api.py b/src/ceph-node-proxy/ceph_node_proxy/api.py
index 25ae03e5195272eafc1f1a604052e5285cd599ee..676d7c1c61d21b0daf75e77079a8b3ea5a589189 100644 (file)
--- a/src/ceph-node-proxy/ceph_node_proxy/api.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/api.py
@@ -63,7 +63,7 @@ class API(Server):
         self.backend = backend
         self.reporter = reporter
         self.config = config
-        self.socket_port = self.config.__dict__['api']['port'] if not port else port
+        self.socket_port = port if port else self.config.get('api', {}).get('port', 9456)
         self.socket_host = addr
         self.subscribe()
 

diff --git a/src/ceph-node-proxy/ceph_node_proxy/basesystem.py b/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
index 65eca55af1f07c9aa02e88e4b97957bdeb363e92..f25b963e6aad40b809bf926ce4b570497187e665 100644 (file)
--- a/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
@@ -1,7 +1,7 @@
 import socket
 from threading import Lock
 from ceph_node_proxy.util import Config, get_logger, BaseThread
-from typing import Dict, Any
+from typing import Dict, Any, Optional, Union
 from ceph_node_proxy.baseclient import BaseClient
 
 
@@ -10,7 +10,7 @@ class BaseSystem(BaseThread):
         super().__init__()
         self.lock: Lock = Lock()
         self._system: Dict = {}
-        self.config: Config = kw.get('config', {})
+        self.config: Optional[Union[Config, Dict[str, Any]]] = kw.get('config')
         self.client: BaseClient
         self.log = get_logger(__name__)
 

diff --git a/src/ceph-node-proxy/ceph_node_proxy/main.py b/src/ceph-node-proxy/ceph_node_proxy/main.py
index e37996a4cc8f9cc7d286fae6901fbfc966757e54..13887112589402d2fbb55aae1cb55a96b5f104e9 100644 (file)
--- a/src/ceph-node-proxy/ceph_node_proxy/main.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/main.py
@@ -3,7 +3,7 @@ from ceph_node_proxy.atollon import AtollonSystem
 from ceph_node_proxy.baseredfishsystem import BaseRedfishSystem
 from ceph_node_proxy.redfishdellsystem import RedfishDellSystem
 from ceph_node_proxy.reporter import Reporter
-from ceph_node_proxy.util import Config, get_logger, http_req, write_tmp_file, CONFIG
+from ceph_node_proxy.util import Config, DEFAULTS, get_logger, http_req, write_tmp_file
 from urllib.error import HTTPError
 from typing import Dict, Any, Optional, Type
 
@@ -42,7 +42,8 @@ class NodeProxyManager:
         self.reporter_endpoint: str = kw.get('reporter_endpoint', '/node-proxy/data')
         self.cephx = {'cephx': {'name': self.cephx_name,
                                 'secret': self.cephx_secret}}
-        self.config = Config('/etc/ceph/node-proxy.yml', config=CONFIG)
+        config_path = kw.get('config_path') or os.environ.get('NODE_PROXY_CONFIG', '/etc/ceph/node-proxy.yml')
+        self.config = Config(config_path, defaults=DEFAULTS)
         self.username: str = ''
         self.password: str = ''
 
@@ -85,7 +86,7 @@ class NodeProxyManager:
             self.log.warning('No oob details could be loaded, exiting...')
             raise SystemExit(1)
         try:
-            vendor = getattr(self.config, 'system', {}).get('vendor', 'generic')
+            vendor = self.config.get('system', {}).get('vendor', 'generic')
             system_cls = REDFISH_SYSTEM_CLASSES.get(vendor, BaseRedfishSystem)
             self.system = system_cls(host=oob_details['host'],
                                      port=oob_details['port'],
@@ -171,7 +172,7 @@ def main() -> None:
 
     args = parser.parse_args()
     if args.debug:
-        CONFIG['logging']['level'] = 10
+        DEFAULTS['logging']['level'] = 10
 
     if not os.path.exists(args.config):
         raise Exception(f'No config file found at provided config path: {args.config}')
@@ -194,13 +195,16 @@ def main() -> None:
     ca_file = write_tmp_file(root_cert,
                              prefix_name='cephadm-endpoint-root-cert')
 
+    config_path = config.get('node_proxy_config') or os.environ.get('NODE_PROXY_CONFIG', '/etc/ceph/node-proxy.yml')
+
     node_proxy_mgr = NodeProxyManager(mgr_host=target_ip,
                                       cephx_name=name,
                                       cephx_secret=keyring,
                                       mgr_agent_port=target_port,
                                       ca_path=ca_file.name,
                                       api_ssl_crt=listener_cert,
-                                      api_ssl_key=listener_key)
+                                      api_ssl_key=listener_key,
+                                      config_path=config_path)
     signal.signal(signal.SIGTERM,
                   lambda signum, frame: handler(signum, frame, node_proxy_mgr))
     node_proxy_mgr.run()

diff --git a/src/ceph-node-proxy/ceph_node_proxy/util.py b/src/ceph-node-proxy/ceph_node_proxy/util.py
index bcee6171a99fd21d07a647f9d5065fb56616cae4..15c20022b633a2e0a48be09f860593e314b5e899 100644 (file)
--- a/src/ceph-node-proxy/ceph_node_proxy/util.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/util.py
@@ -12,7 +12,7 @@ from urllib.request import urlopen, Request
 from typing import Dict, Callable, Any, Optional, MutableMapping, Tuple, Union
 
 
-CONFIG: Dict[str, Any] = {
+DEFAULTS: Dict[str, Any] = {
     'reporter': {
         'check_interval': 5,
         'push_data_max_retries': 30,
@@ -27,14 +27,36 @@ CONFIG: Dict[str, Any] = {
     },
     'logging': {
         'level': logging.INFO,
-    }
+    },
 }
 
 
+def _deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
+    result = dict(base)
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            result[key] = _deep_merge(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+
+def load_config(
+    path: str,
+    defaults: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    defaults = defaults or {}
+    if not os.path.exists(path):
+        return _deep_merge({}, defaults)
+    with open(path, 'r') as f:
+        loaded = yaml.safe_load(f) or {}
+    return _deep_merge(defaults, loaded)
+
+
 def get_logger(name: str, level: Union[int, str] = logging.NOTSET) -> logging.Logger:
     log_level: Union[int, str] = level
     if log_level == logging.NOTSET:
-        log_level = CONFIG['logging']['level']
+        log_level = DEFAULTS['logging']['level']
     logger = logging.getLogger(name)
     logger.setLevel(log_level)
     handler = logging.StreamHandler()
@@ -52,32 +74,25 @@ logger = get_logger(__name__)
 
 
 class Config:
-    def __init__(self,
-                 config_file: str = '/etc/ceph/node-proxy.yaml',
-                 config: Dict[str, Any] = {}) -> None:
-        self.config_file = config_file
-        self.config = config
-
-        self.load_config()
-
-    def load_config(self) -> None:
-        if os.path.exists(self.config_file):
-            with open(self.config_file, 'r') as f:
-                self.config = yaml.safe_load(f)
-        else:
-            self.config = self.config
-
-        for k, v in self.config.items():
-            if k not in self.config.keys():
-                self.config[k] = v
-
-        for k, v in self.config.items():
-            setattr(self, k, v)
-
-    def reload(self, config_file: str = '') -> None:
-        if config_file != '':
-            self.config_file = config_file
-        self.load_config()
+    def __init__(
+        self,
+        path: str,
+        defaults: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        self.path = path
+        self.defaults = defaults or {}
+        self._data = load_config(self.path, self.defaults)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        return self._data.get(key, default)
+
+    def __getitem__(self, key: str) -> Any:
+        return self._data[key]
+
+    def reload(self, path: Optional[str] = None) -> None:
+        if path is not None:
+            self.path = path
+        self._data = load_config(self.path, self.defaults)
 
 
 class BaseThread(threading.Thread):