From: Afreen Misbah Date: Thu, 25 Jun 2026 08:13:38 +0000 (+0530) Subject: mgr/prometheus: use orchestrator API for node-proxy hardware metrics X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6bb07c6e5393daeb72fe0cf3832955721e7b4b18;p=ceph.git mgr/prometheus: use orchestrator API for node-proxy hardware metrics The hardware metrics exporter was reading cephadm's private KV store directly via get_store_prefix('mgr/cephadm/node_proxy/data'). This had two problems: 1. get_store_prefix() is module-scoped, so from the prometheus module it searched under prometheus's own namespace instead of cephadm's, resulting in zero metrics being exported despite metric definitions appearing at /metrics. 2. The firmware key was accessed as 'firmwares' (plural) but the stored field is 'firmware' (singular), causing all firmware version metrics to be silently empty. Use node_proxy_fullreport() and node_proxy_firmware() via the OrchestratorClientMixin instead. This routes through cephadm's NodeProxyCache which handles KV access and firmware key compat correctly. Follows the same pattern as set_cephadm_daemon_status_metrics() and get_smb_metadata(). Signed-off-by: Afreen Misbah Assisted-by: Claude Signed-off-by: Afreen Misbah --- diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py index 7e334b8d633..f2d8a230611 100644 --- a/src/pybind/mgr/prometheus/module.py +++ b/src/pybind/mgr/prometheus/module.py @@ -2065,112 +2065,99 @@ class Module(MgrModule, OrchestratorClientMixin): @profile_method(True) def get_hardware_metrics(self) -> None: - """Collect hardware metrics from node-proxy KV store""" - NODE_PROXY_CACHE_PREFIX = 'mgr/cephadm/node_proxy' + if not self.orch_is_available(): + return try: - for key, value in self.get_store_prefix(f'{NODE_PROXY_CACHE_PREFIX}/data').items(): - host = key.split('/')[-1] - try: - data = json.loads(value) - except json.JSONDecodeError: - self.log.error(f"Failed to decode node-proxy data for host {host}") - continue + report = raise_if_exception(self.node_proxy_fullreport()) + firmware_report = raise_if_exception(self.node_proxy_firmware()) + except Exception as e: + self.log.debug(f"node-proxy data not available: {e}") + return - status = data.get('status', {}) - - # Storage metrics - for sys_id, devices in status.get('storage', {}).items(): - for device_id, device in devices.items(): - capacity = device.get('capacity_bytes', 0) - labels = ( - host, - device_id, - device.get('model', 'unknown'), - device.get('protocol', 'unknown') - ) - self.metrics['node_proxy_storage_capacity_bytes'].set(capacity, labels) - - # CPU metrics - for sys_id, cpus in status.get('processors', {}).items(): - for cpu_id, cpu in cpus.items(): - cores = cpu.get('total_cores', 0) - labels = ( - host, - cpu_id, - cpu.get('manufacturer', 'unknown'), - cpu.get('model', 'unknown') - ) - self.metrics['node_proxy_cpu_cores'].set(cores, labels) - - # Memory metrics - for sys_id, dimms in status.get('memory', {}).items(): - for dimm_id, dimm in dimms.items(): - capacity = dimm.get('capacity_mi_b', 0) - labels = ( - host, - dimm_id, - dimm.get('memory_device_type', 'unknown'), - dimm.get('manufacturer', 'unknown') - ) - self.metrics['node_proxy_memory_capacity_mib'].set(capacity, labels) - - # Health metrics for all component types - for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network']: - for sys_id, components in status.get(category, {}).items(): - for comp_id, comp in components.items(): - health_str = comp.get('status', {}).get('health', 'Unknown') - if health_str == 'OK': - health_value = 0 - elif health_str in ['Warning', 'Degraded']: - health_value = 1 - else: - health_value = 2 - - labels = (host, comp_id, category) - self.metrics['node_proxy_health'].set(health_value, labels) - - # Temperature metrics - for sys_id, sensors in status.get('temperatures', {}).items(): - for sensor_id, sensor in sensors.items(): - reading = sensor.get('reading') - sensor_name = sensor.get('name', sensor_id) - # Skip absent/unknown sensors - if reading is None or reading == 'unknown': - continue - try: - temp_value = float(reading) - labels = (host, sensor_name) - self.metrics['node_proxy_temperature_celsius'].set(temp_value, labels) - except (ValueError, TypeError): - continue + for host, data in report.items(): + status = data.get('status', {}) + + for sys_id, devices in status.get('storage', {}).items(): + for device_id, device in devices.items(): + capacity = device.get('capacity_bytes', 0) + labels = ( + host, + device_id, + device.get('model', 'unknown'), + device.get('protocol', 'unknown') + ) + self.metrics['node_proxy_storage_capacity_bytes'].set(capacity, labels) + + for sys_id, cpus in status.get('processors', {}).items(): + for cpu_id, cpu in cpus.items(): + cores = cpu.get('total_cores', 0) + labels = ( + host, + cpu_id, + cpu.get('manufacturer', 'unknown'), + cpu.get('model', 'unknown') + ) + self.metrics['node_proxy_cpu_cores'].set(cores, labels) + + for sys_id, dimms in status.get('memory', {}).items(): + for dimm_id, dimm in dimms.items(): + capacity = dimm.get('capacity_mi_b', 0) + labels = ( + host, + dimm_id, + dimm.get('memory_device_type', 'unknown'), + dimm.get('manufacturer', 'unknown') + ) + self.metrics['node_proxy_memory_capacity_mib'].set(capacity, labels) + + for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network']: + for sys_id, components in status.get(category, {}).items(): + for comp_id, comp in components.items(): + health_str = comp.get('status', {}).get('health', 'Unknown') + if health_str == 'OK': + health_value = 0 + elif health_str in ['Warning', 'Degraded']: + health_value = 1 + else: + health_value = 2 - # Fan RPM metrics - for sys_id, fans_data in status.get('fans', {}).items(): - for fan_id, fan in fans_data.items(): - # Redfish uses 'Reading' field per Thermal schema - # node-proxy normalizes it to lowercase 'reading' - rpm = fan.get('reading') - fan_name = fan.get('name', fan_id) - if rpm is None or rpm == 'unknown': - continue - try: - rpm_value = float(rpm) - labels = (host, fan_name) - self.metrics['node_proxy_fan_rpm'].set(rpm_value, labels) - except (ValueError, TypeError): - continue + labels = (host, comp_id, category) + self.metrics['node_proxy_health'].set(health_value, labels) - # Firmware version metrics - firmwares = data.get('firmwares', {}) - for fw_id, fw_info in firmwares.items(): - component = fw_info.get('name', fw_id) - version = fw_info.get('version', 'unknown') - if version and version != 'unknown': - labels = (host, component, str(version)) - self.metrics['node_proxy_firmware_info'].set(1, labels) - except Exception as e: - self.log.error(f"Failed to collect hardware metrics: {str(e)}") + for sys_id, sensors in status.get('temperatures', {}).items(): + for sensor_id, sensor in sensors.items(): + reading = sensor.get('reading') + sensor_name = sensor.get('name', sensor_id) + if reading is None or reading == 'unknown': + continue + try: + temp_value = float(reading) + labels = (host, sensor_name) + self.metrics['node_proxy_temperature_celsius'].set(temp_value, labels) + except (ValueError, TypeError): + continue + + for sys_id, fans_data in status.get('fans', {}).items(): + for fan_id, fan in fans_data.items(): + rpm = fan.get('reading') + fan_name = fan.get('name', fan_id) + if rpm is None or rpm == 'unknown': + continue + try: + rpm_value = float(rpm) + labels = (host, fan_name) + self.metrics['node_proxy_fan_rpm'].set(rpm_value, labels) + except (ValueError, TypeError): + continue + + for host, fw_data in firmware_report.items(): + for fw_id, fw_info in fw_data.items(): + component = fw_info.get('name', fw_id) + version = fw_info.get('version', 'unknown') + if version and version != 'unknown': + labels = (host, component, str(version)) + self.metrics['node_proxy_firmware_info'].set(1, labels) @profile_method(True) def collect(self) -> str: