From: Afreen Misbah Date: Tue, 30 Jun 2026 14:09:13 +0000 (+0530) Subject: mgr/prometheus: refactor hardware metrics X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=be324304db45f37d3a4027ab568c5010003ef486;p=ceph.git mgr/prometheus: refactor hardware metrics - export `firmware_version` labels from `node_proxy_storage_capacity_bytes` metrics to be used in device firmware panel - improve iterations for performance - dropping redundant node_proxy_firmware() RPC; firmware data is already present in the fullreport response, and removing unnecessary loops. - replace health if/elif chain with HEALTH_STATUS_MAP dict lookup - rename metrics to ceph_hardware_*, fix component labels and add tests - added unit test cases - add serial name, slot info to capacity metric - fix health metrics showing ID instead of component name Signed-off-by: Afreen Misbah --- diff --git a/monitoring/ceph-mixin/dashboards/hardware.libsonnet b/monitoring/ceph-mixin/dashboards/hardware.libsonnet index 1fd0c02f0ee..583d93bb81b 100644 --- a/monitoring/ceph-mixin/dashboards/hardware.libsonnet +++ b/monitoring/ceph-mixin/dashboards/hardware.libsonnet @@ -47,7 +47,7 @@ local g = import 'grafonnet/grafana.libsonnet'; $.addTemplateSchema( 'hostname', '$datasource', - 'label_values(ceph_node_proxy_health, hostname)', + 'label_values(ceph_hardware_health, hostname)', 1, false, 1, @@ -59,14 +59,14 @@ local g = import 'grafonnet/grafana.libsonnet'; g.template.new( 'fan_speeds', '$datasource', - 'label_values(ceph_node_proxy_fan_rpm{hostname=~"$hostname"},fan_name)', + 'label_values(ceph_hardware_fan_rpm{hostname=~"$hostname"},fan_name)', label='', refresh='load', includeAll=true, multi=true, allValues='', sort=0, - regex='', + regex='/.*TACH_OUT/', hide=2 ) ) @@ -81,7 +81,7 @@ local g = import 'grafonnet/grafana.libsonnet'; gridPosition={ h: 4, w: 4, x: 0, y: 1 } ) .addTarget($.addTargetSchema( - 'max(ceph_node_proxy_health)', + 'max(ceph_hardware_health)', legendFormat='__auto' )) + { @@ -115,7 +115,7 @@ local g = import 'grafonnet/grafana.libsonnet'; { color: 'red', value: 85 }, ]) .addTarget($.addTargetSchema( - 'max(ceph_node_proxy_temperature_celsius{sensor_name=~".*CPU_TEMP"})', + 'max(ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"})', legendFormat='__auto' )), @@ -126,7 +126,7 @@ local g = import 'grafonnet/grafana.libsonnet'; gridPos={ x: 7, y: 1, w: 3, h: 8 } ) .addTarget($.addTargetSchema( - 'count by(version) (ceph_node_proxy_firmware_info{component=~".*BMC.*"})', + 'count by(version) (ceph_hardware_firmware_info{component=~".*BMC.*"})', legendFormat='{{version}}' )) + { @@ -149,7 +149,7 @@ local g = import 'grafonnet/grafana.libsonnet'; gridPos={ x: 10, y: 1, w: 3, h: 8 } ) .addTarget($.addTargetSchema( - 'count by(version) (ceph_node_proxy_firmware_info{component=~".*BIOS.*"})', + 'count by(version) (ceph_hardware_firmware_info{component=~".*BIOS.*"})', legendFormat='{{version}}' )) + { @@ -172,7 +172,7 @@ local g = import 'grafonnet/grafana.libsonnet'; gridPos={ x: 13, y: 1, w: 6, h: 8 } ) .addTarget($.addTargetSchema( - 'count by(model) (ceph_node_proxy_storage_capacity_bytes)', + 'count by(model) (ceph_hardware_storage_capacity_bytes)', legendFormat='{{model}}' )) + { @@ -195,8 +195,8 @@ local g = import 'grafonnet/grafana.libsonnet'; gridPos={ x: 19, y: 1, w: 5, h: 8 } ) .addTarget($.addTargetSchema( - 'count by(version) (ceph_node_proxy_firmware_info{component=~".*Drive.*"})', - legendFormat='{{version}}', + 'count by(firmware_version) (ceph_hardware_storage_capacity_bytes{firmware_version!="unknown"})', + legendFormat='{{firmware_version}}', instant=true )) + { @@ -219,7 +219,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 2, x: 0, y: 5 } ).addTarget($.addTargetSchema( - 'count(count by(hostname) (ceph_node_proxy_health))', + 'count(count by(hostname) (ceph_hardware_health))', legendFormat='__auto' )), @@ -230,7 +230,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 2, x: 2, y: 5 } ).addTarget($.addTargetSchema( - 'count(ceph_node_proxy_storage_capacity_bytes)', + 'count(ceph_hardware_storage_capacity_bytes)', legendFormat='__auto' )), @@ -249,7 +249,7 @@ local g = import 'grafonnet/grafana.libsonnet'; { color: 'red', value: 80 }, ]) .addTarget($.addTargetSchema( - 'max(ceph_node_proxy_temperature_celsius{sensor_name=~"NVME.*_TEMP"})', + 'max(ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"})', legendFormat='__auto' )), ] }, @@ -263,7 +263,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 2 } ).addTarget($.addTargetSchema( - 'max(ceph_node_proxy_health{hostname=~"$hostname",category="power"})', + 'max(ceph_hardware_health{hostname=~"$hostname",category="power"})', legendFormat='__auto' )) + { @@ -286,9 +286,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 2, y: 2 } ).addTarget($.addTargetSchema( - 'max(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})', + 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})', legendFormat='__auto' - )), + )) + + { + fieldConfig+: { defaults+: { thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'red', value: 80 }] } } }, + options+: { colorMode: 'none', graphMode: 'area' }, + }, // CPU Temperature $.addStatPanel( @@ -297,9 +301,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 5, y: 2 } ).addTarget($.addTargetSchema( - 'avg(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})', + 'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})', legendFormat='__auto' - )), + )) + + { + fieldConfig+: { defaults+: { max: 85, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 80 }, { color: 'semi-dark-red', value: 90 }] } } }, + options+: { colorMode: 'none', graphMode: 'area' }, + }, // DIMM Temperature $.addStatPanel( @@ -308,9 +316,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 8, y: 2 } ).addTarget($.addTargetSchema( - 'max(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})', + 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})', legendFormat='__auto' - )), + )) + + { + fieldConfig+: { defaults+: { max: 88, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'yellow', value: 70 }, { color: 'semi-dark-red', value: 80 }] } } }, + options+: { colorMode: 'none', graphMode: 'area' }, + }, // PSU Fans $.addStatPanel( @@ -319,9 +331,10 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 11, y: 2 } ).addTarget($.addTargetSchema( - 'count(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})', + 'count(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})', legendFormat='__auto' - )), + )) + + { options+: { colorMode: 'none', graphMode: 'none' } }, // AVG PSU Temperature $.addStatPanel( @@ -330,9 +343,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 14, y: 2 } ).addTarget($.addTargetSchema( - 'avg(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})', + 'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})', legendFormat='__auto' - )), + )) + + { + fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' }, max: 100, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'yellow', value: 80 }, { color: 'semi-dark-red', value: 90 }] } } }, + options+: { colorMode: 'none', graphMode: 'area' }, + }, // NVMe Drives $.addStatPanel( @@ -341,9 +358,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 17, y: 2 } ).addTarget($.addTargetSchema( - 'count(ceph_node_proxy_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})', + 'count(ceph_hardware_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})', legendFormat='__auto' - )), + )) + + { + fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'semi-dark-red', value: 78 }] } } }, + options+: { colorMode: 'none', graphMode: 'none' }, + }, // NVMe Temperature $.addStatPanel( @@ -352,9 +373,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 20, y: 2 } ).addTarget($.addTargetSchema( - 'max(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})', + 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})', legendFormat='__auto' - )), + )) + + { + fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'thresholds' }, max: 85, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'semi-dark-red', value: 78 }] } } }, + options+: { colorMode: 'none', graphMode: 'area' }, + }, // Network $.addStatPanel( @@ -363,7 +388,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 5 } ).addTarget($.addTargetSchema( - 'max(ceph_node_proxy_health{hostname=~"$hostname",category="network"})', + 'max(ceph_hardware_health{hostname=~"$hostname",category="network"})', legendFormat='__auto' )) + { @@ -386,7 +411,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 8 } ).addTarget($.addTargetSchema( - 'max(ceph_node_proxy_health{hostname=~"$hostname",category="fans"})', + 'max(ceph_hardware_health{hostname=~"$hostname",category="fans"})', legendFormat='__auto' )) + { @@ -409,7 +434,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 11 } ).addTarget($.addTargetSchema( - 'max(ceph_node_proxy_health{hostname=~"$hostname",category="storage"})', + 'max(ceph_hardware_health{hostname=~"$hostname",category="storage"})', legendFormat='__auto' )) + { @@ -441,7 +466,7 @@ local g = import 'grafonnet/grafana.libsonnet'; }, ) .addTarget($.addTargetSchema( - 'ceph_node_proxy_storage_capacity_bytes{hostname=~"$hostname"}', + 'ceph_hardware_storage_capacity_bytes{hostname=~"$hostname"}', legendFormat='__auto', instant=true ) + { format: 'table' }) @@ -483,7 +508,7 @@ local g = import 'grafonnet/grafana.libsonnet'; }, ) .addTarget($.addTargetSchema( - 'ceph_node_proxy_firmware_info{hostname=~"$hostname"}', + 'ceph_hardware_firmware_info{hostname=~"$hostname"}', legendFormat='__auto', instant=true ) + { format: 'table' }) @@ -518,10 +543,12 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 5, w: 4, x: 0, y: 3 } ).addTarget($.addTargetSchema( - 'ceph_node_proxy_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}', + 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}', legendFormat='{{fan_name}}' )) + { + fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } }, + options+: { colorMode: 'none', graphMode: 'area' }, repeat: 'fan_speeds', repeatDirection: 'h', maxPerRow: 6, @@ -534,9 +561,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 5, w: 4, x: 0, y: 8 } ).addTarget($.addTargetSchema( - 'ceph_node_proxy_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}', + 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}', legendFormat='{{fan_name}}' - )), + )) + + { + fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } }, + options+: { colorMode: 'none', graphMode: 'area' }, + }, // PSU2 $.addStatPanel( @@ -545,9 +576,13 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 5, w: 4, x: 4, y: 8 } ).addTarget($.addTargetSchema( - 'ceph_node_proxy_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}', + 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}', legendFormat='{{fan_name}}' - )), + )) + + { + fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } }, + options+: { colorMode: 'none', graphMode: 'area' }, + }, ] }, // Row 4: Temperature History @@ -559,7 +594,7 @@ local g = import 'grafonnet/grafana.libsonnet'; format='celsius', ).addTarget( g.prometheus.target( - 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"}', + 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"}', legendFormat='{{sensor_name}}' ) ).addTarget( @@ -580,7 +615,7 @@ local g = import 'grafonnet/grafana.libsonnet'; format='celsius', ).addTarget( g.prometheus.target( - 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"}', + 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"}', legendFormat='{{sensor_name}}' ) ).addTarget( @@ -601,7 +636,7 @@ local g = import 'grafonnet/grafana.libsonnet'; format='celsius', ).addTarget( g.prometheus.target( - 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"}', + 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"}', legendFormat='{{sensor_name}}' ) ).addTarget( @@ -622,7 +657,7 @@ local g = import 'grafonnet/grafana.libsonnet'; format='celsius', ).addTarget( g.prometheus.target( - 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*_TEMP"}', + 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*_TEMP"}', legendFormat='{{sensor_name}}' ) ).addTarget( @@ -643,10 +678,11 @@ local g = import 'grafonnet/grafana.libsonnet'; g.graphPanel.new( title='AVG PSU Fan Speed', datasource='$datasource', - format='none', + format='locale', + fill=1, ).addTarget( g.prometheus.target( - 'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})', + 'avg(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})', legendFormat='PSU Fans' ) ) @@ -655,14 +691,15 @@ local g = import 'grafonnet/grafana.libsonnet'; description: 'AVG across both PSUs', }, - // AVG Cooling Fan Speed + // AVG Cooling Fan RPMs g.graphPanel.new( - title='AVG Cooling Fan Speed', + title='AVG Cooling Fan RPMs', datasource='$datasource', - format='none', + format='locale', + fill=1, ).addTarget( g.prometheus.target( - 'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name!~"PSU.*"})', + 'avg(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name!~"PSU.*"})', legendFormat='System Fans' ) ) diff --git a/monitoring/ceph-mixin/dashboards_out/hardware.json b/monitoring/ceph-mixin/dashboards_out/hardware.json index b6afffb16fc..263ac97ccfe 100644 --- a/monitoring/ceph-mixin/dashboards_out/hardware.json +++ b/monitoring/ceph-mixin/dashboards_out/hardware.json @@ -117,7 +117,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_health)", + "expr": "max(ceph_hardware_health)", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -180,7 +180,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_temperature_celsius{sensor_name=~\".*CPU_TEMP\"})", + "expr": "max(ceph_hardware_temperature_celsius{sensor_name=~\".*CPU_TEMP\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -235,7 +235,7 @@ }, "targets": [ { - "expr": "count by(version) (ceph_node_proxy_firmware_info{component=~\".*BMC.*\"})", + "expr": "count by(version) (ceph_hardware_firmware_info{component=~\".*BMC.*\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{version}}", @@ -289,7 +289,7 @@ }, "targets": [ { - "expr": "count by(version) (ceph_node_proxy_firmware_info{component=~\".*BIOS.*\"})", + "expr": "count by(version) (ceph_hardware_firmware_info{component=~\".*BIOS.*\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{version}}", @@ -342,7 +342,7 @@ }, "targets": [ { - "expr": "count by(model) (ceph_node_proxy_storage_capacity_bytes)", + "expr": "count by(model) (ceph_hardware_storage_capacity_bytes)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{model}}", @@ -395,11 +395,11 @@ }, "targets": [ { - "expr": "count by(version) (ceph_node_proxy_firmware_info{component=~\".*Drive.*\"})", + "expr": "count by(firmware_version) (ceph_hardware_storage_capacity_bytes{firmware_version!=\"unknown\"})", "format": "time_series", "instant": true, "intervalFactor": 1, - "legendFormat": "{{version}}", + "legendFormat": "{{firmware_version}}", "refId": "A" } ], @@ -447,7 +447,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "count(count by(hostname) (ceph_node_proxy_health))", + "expr": "count(count by(hostname) (ceph_hardware_health))", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -499,7 +499,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "count(ceph_node_proxy_storage_capacity_bytes)", + "expr": "count(ceph_hardware_storage_capacity_bytes)", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -562,7 +562,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_temperature_celsius{sensor_name=~\"NVME.*_TEMP\"})", + "expr": "max(ceph_hardware_temperature_celsius{sensor_name=~\"NVME.*_TEMP\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -662,7 +662,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"power\"})", + "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"power\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -684,7 +684,19 @@ "mappings": [ ], "thresholds": { "mode": "absolute", - "steps": [ ] + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 80 + } + ] }, "unit": "celsius" } @@ -699,7 +711,7 @@ "links": [ ], "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -714,7 +726,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"})", + "expr": "max(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -734,9 +746,23 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "max": 85, + "min": 0, "thresholds": { "mode": "absolute", - "steps": [ ] + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "semi-dark-red", + "value": 90 + } + ] }, "unit": "celsius" } @@ -751,7 +777,7 @@ "links": [ ], "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -766,7 +792,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "avg(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"})", + "expr": "avg(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -786,9 +812,23 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "max": 88, + "min": 0, "thresholds": { "mode": "absolute", - "steps": [ ] + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] }, "unit": "celsius" } @@ -803,7 +843,7 @@ "links": [ ], "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -818,7 +858,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"})", + "expr": "max(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -870,7 +910,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "count(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})", + "expr": "count(ceph_hardware_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -887,12 +927,30 @@ "description": "", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "decimals": 0, "links": [ ], "mappings": [ ], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", - "steps": [ ] + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "semi-dark-red", + "value": 90 + } + ] }, "unit": "celsius" } @@ -907,7 +965,7 @@ "links": [ ], "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -922,7 +980,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "avg(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"PSU.*_TEMP.*\"})", + "expr": "avg(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"PSU.*_TEMP.*\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -939,12 +997,28 @@ "description": "", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, "decimals": 0, "links": [ ], "mappings": [ ], "thresholds": { "mode": "absolute", - "steps": [ ] + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "semi-dark-red", + "value": 78 + } + ] }, "unit": "short" } @@ -974,7 +1048,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "count(ceph_node_proxy_storage_capacity_bytes{hostname=~\"$hostname\", protocol=\"NVMe\"})", + "expr": "count(ceph_hardware_storage_capacity_bytes{hostname=~\"$hostname\", protocol=\"NVMe\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -991,12 +1065,30 @@ "description": "", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, "decimals": 0, "links": [ ], "mappings": [ ], + "max": 85, + "min": 0, "thresholds": { "mode": "absolute", - "steps": [ ] + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "semi-dark-red", + "value": 78 + } + ] }, "unit": "celsius" } @@ -1011,7 +1103,7 @@ "links": [ ], "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -1026,7 +1118,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*\"})", + "expr": "max(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -1106,7 +1198,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"network\"})", + "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"network\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -1186,7 +1278,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"fans\"})", + "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"fans\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -1266,7 +1358,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"storage\"})", + "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"storage\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -1320,7 +1412,7 @@ "styles": [ ], "targets": [ { - "expr": "ceph_node_proxy_storage_capacity_bytes{hostname=~\"$hostname\"}", + "expr": "ceph_hardware_storage_capacity_bytes{hostname=~\"$hostname\"}", "format": "table", "instant": true, "intervalFactor": 1, @@ -1397,7 +1489,7 @@ "styles": [ ], "targets": [ { - "expr": "ceph_node_proxy_firmware_info{hostname=~\"$hostname\"}", + "expr": "ceph_hardware_firmware_info{hostname=~\"$hostname\"}", "format": "table", "instant": true, "intervalFactor": 1, @@ -1456,6 +1548,10 @@ "description": "", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "decimals": 0, "links": [ ], "mappings": [ ], @@ -1477,7 +1573,7 @@ "maxPerRow": 6, "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -1494,7 +1590,7 @@ "repeatDirection": "h", "targets": [ { - "expr": "ceph_node_proxy_fan_rpm{hostname=~\"$hostname\",fan_name=~\"$fan_speeds\"}", + "expr": "ceph_hardware_fan_rpm{hostname=~\"$hostname\",fan_name=~\"$fan_speeds\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{fan_name}}", @@ -1511,6 +1607,10 @@ "description": "", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "decimals": 0, "links": [ ], "mappings": [ ], @@ -1531,7 +1631,7 @@ "links": [ ], "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -1546,7 +1646,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "ceph_node_proxy_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU1.*\"}", + "expr": "ceph_hardware_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU1.*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{fan_name}}", @@ -1563,6 +1663,10 @@ "description": "", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "decimals": 0, "links": [ ], "mappings": [ ], @@ -1583,7 +1687,7 @@ "links": [ ], "options": { "colorMode": "none", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -1598,7 +1702,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "ceph_node_proxy_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU2.*\"}", + "expr": "ceph_hardware_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU2.*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{fan_name}}", @@ -1678,7 +1782,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"}", + "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sensor_name}}", @@ -1765,7 +1869,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"}", + "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sensor_name}}", @@ -1852,7 +1956,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"}", + "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sensor_name}}", @@ -1939,7 +2043,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*_TEMP\"}", + "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*_TEMP\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sensor_name}}", @@ -2040,7 +2144,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})", + "expr": "avg(ceph_hardware_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "PSU Fans", @@ -2066,7 +2170,7 @@ }, "yaxes": [ { - "format": "none", + "format": "locale", "label": null, "logBase": 1, "max": null, @@ -2074,7 +2178,7 @@ "show": true }, { - "format": "none", + "format": "locale", "label": null, "logBase": 1, "max": null, @@ -2125,7 +2229,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name!~\"PSU.*\"})", + "expr": "avg(ceph_hardware_fan_rpm{hostname=~\"$hostname\", fan_name!~\"PSU.*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "System Fans", @@ -2135,7 +2239,7 @@ "thresholds": [ ], "timeFrom": null, "timeShift": null, - "title": "AVG Cooling Fan Speed", + "title": "AVG Cooling Fan RPMs", "tooltip": { "shared": true, "sort": 0, @@ -2151,7 +2255,7 @@ }, "yaxes": [ { - "format": "none", + "format": "locale", "label": null, "logBase": 1, "max": null, @@ -2159,7 +2263,7 @@ "show": true }, { - "format": "none", + "format": "locale", "label": null, "logBase": 1, "max": null, @@ -2231,7 +2335,7 @@ "multi": false, "name": "hostname", "options": [ ], - "query": "label_values(ceph_node_proxy_health, hostname)", + "query": "label_values(ceph_hardware_health, hostname)", "refresh": 1, "regex": "", "sort": 1, @@ -2251,9 +2355,9 @@ "multi": true, "name": "fan_speeds", "options": [ ], - "query": "label_values(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\"},fan_name)", + "query": "label_values(ceph_hardware_fan_rpm{hostname=~\"$hostname\"},fan_name)", "refresh": 1, - "regex": "", + "regex": "/.*TACH_OUT/", "sort": 0, "tagValuesQuery": "", "tags": [ ], diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py index 06002b7b195..4b353d2314a 100644 --- a/src/pybind/mgr/prometheus/module.py +++ b/src/pybind/mgr/prometheus/module.py @@ -140,20 +140,41 @@ NUM_OBJECTS = ['degraded', 'misplaced', 'unfound'] SMB_METADATA = ('smb_version', 'volume', 'subvolume_group', 'subvolume', 'netbiosname', 'share') -NODE_PROXY_STORAGE_LABELS = ('hostname', 'device', 'model', 'protocol') +HW_STORAGE_LABELS = ('hostname', 'device', 'model', 'protocol', 'firmware_version', 'slot', 'serial_number') -NODE_PROXY_CPU_LABELS = ('hostname', 'cpu', 'manufacturer', 'model') +HW_CPU_LABELS = ('hostname', 'cpu', 'manufacturer', 'model', 'total_threads') -NODE_PROXY_MEMORY_LABELS = ('hostname', 'dimm', 'type', 'manufacturer') +HW_MEMORY_LABELS = ('hostname', 'dimm', 'type') MIB_TO_BYTES = 1048576 -NODE_PROXY_HEALTH_LABELS = ('hostname', 'component', 'category') +HW_HEALTH_LABELS = ('hostname', 'component', 'category') -NODE_PROXY_TEMP_LABELS = ('hostname', 'sensor_name') +HW_TEMP_LABELS = ('hostname', 'sensor_name') -NODE_PROXY_FAN_LABELS = ('hostname', 'fan_name') +HW_FAN_LABELS = ('hostname', 'fan_name') -NODE_PROXY_FIRMWARE_LABELS = ('hostname', 'component', 'version') +HW_FIRMWARE_LABELS = ('hostname', 'component', 'version') + +HEALTH_STATUS_MAP = { + 'OK': 0, + 'Warning': 1, + 'Degraded': 1, + 'Error': 2, + 'Critical': 2, +} + +SENSOR_METRICS = { + 'fans': { + 'metric': 'hardware_fan_rpm', + 'description': 'Hardware fan speed in RPM', + 'labels': HW_FAN_LABELS, + }, + 'temperatures': { + 'metric': 'hardware_temperature_celsius', + 'description': 'Hardware temperature sensor reading in Celsius', + 'labels': HW_TEMP_LABELS, + }, +} CEPHADM_DAEMON_STATUS = ('service_type', 'daemon_name', 'hostname', 'service_name') @@ -1013,53 +1034,47 @@ class Module(MgrModule, OrchestratorClientMixin): check.description, ) - metrics['node_proxy_storage_capacity_bytes'] = Metric( + metrics['hardware_storage_capacity_bytes'] = Metric( 'gauge', - 'node_proxy_storage_capacity_bytes', + 'hardware_storage_capacity_bytes', 'Storage device capacity in bytes', - NODE_PROXY_STORAGE_LABELS + HW_STORAGE_LABELS ) - metrics['node_proxy_cpu_cores'] = Metric( + metrics['hardware_cpu_cores'] = Metric( 'gauge', - 'node_proxy_cpu_cores', + 'hardware_cpu_cores', 'Total CPU cores', - NODE_PROXY_CPU_LABELS + HW_CPU_LABELS ) - metrics['node_proxy_memory_capacity_bytes'] = Metric( + metrics['hardware_memory_capacity_bytes'] = Metric( 'gauge', - 'node_proxy_memory_capacity_bytes', + 'hardware_memory_capacity_bytes', 'Memory capacity in bytes', - NODE_PROXY_MEMORY_LABELS + HW_MEMORY_LABELS ) - metrics['node_proxy_health'] = Metric( + metrics['hardware_health'] = Metric( 'gauge', - 'node_proxy_health', + 'hardware_health', 'Hardware component health status (0=OK, 1=Warning, 2=Error)', - NODE_PROXY_HEALTH_LABELS - ) - - metrics['node_proxy_temperature_celsius'] = Metric( - 'gauge', - 'node_proxy_temperature_celsius', - 'Hardware temperature sensor reading in Celsius', - NODE_PROXY_TEMP_LABELS + HW_HEALTH_LABELS ) - metrics['node_proxy_fan_rpm'] = Metric( - 'gauge', - 'node_proxy_fan_rpm', - 'Hardware fan speed in RPM', - NODE_PROXY_FAN_LABELS - ) + for sensor in SENSOR_METRICS.values(): + metrics[sensor['metric']] = Metric( + 'gauge', + sensor['metric'], + sensor['description'], + sensor['labels'] + ) - metrics['node_proxy_firmware_info'] = Metric( + metrics['hardware_firmware_info'] = Metric( 'gauge', - 'node_proxy_firmware_info', + 'hardware_firmware_info', 'Firmware version information (value is always 1, version in label)', - NODE_PROXY_FIRMWARE_LABELS + HW_FIRMWARE_LABELS ) return metrics @@ -2064,111 +2079,139 @@ class Module(MgrModule, OrchestratorClientMixin): except Exception as e: self.log.error(f"Failed to get SMB metadata: {str(e)}") + def _hw_get_health_value(self, status_val, hostname='', comp_id='', category=''): + """Map a Redfish health status to a numeric value (0=OK, 1=Warning, 2=Error).""" + if isinstance(status_val, dict): + health_str = status_val.get('health', 'Unknown') + elif isinstance(status_val, str): + health_str = status_val + else: + health_str = 'Unknown' + + value = HEALTH_STATUS_MAP.get(health_str) + if value is None: + self.log.warning( + f"node-proxy: unexpected health status " + f"'{health_str}' for {category}/{comp_id} " + f"on {hostname}, skipping" + ) + return value + + def _hw_set_health_metric(self, status_val, hostname, comp_id, category): + """Set ceph_hardware_health gauge for a single component.""" + health_value = self._hw_get_health_value( + status_val, hostname, comp_id, category + ) + if health_value is not None: + self.metrics['hardware_health'].set( + health_value, (hostname, comp_id, category) + ) + + def _hw_iter_components(self, status, category): + """Yield (comp_id, comp_dict) pairs from nested status[category] dicts.""" + for components in status.get(category, {}).values(): + for comp_id, comp in components.items(): + if isinstance(comp, dict): + yield comp_id, comp + + def _hw_set_sensor_metric(self, comp, comp_id, hostname, category, metric_key): + """Set sensor value (temperature/fan) and health gauges using the sensor name.""" + name = comp.get('name', comp_id) + reading = comp.get('reading') + if reading is not None and reading != 'unknown': + try: + self.metrics[metric_key].set(float(reading), (hostname, name)) + except (ValueError, TypeError): + pass + self._hw_set_health_metric(comp.get('status', {}), hostname, name, category) + + def _process_storage(self, status, hostname): + """Set storage capacity and health metrics per drive.""" + for device_id, device in self._hw_iter_components(status, 'storage'): + capacity = device.get('capacity_bytes', 0) + labels = ( + hostname, + device_id, + device.get('model', 'unknown'), + device.get('protocol', 'unknown'), + device.get('firmware_version', 'unknown'), + device.get('slot', 'unknown'), + device.get('serial_number', 'unknown') + ) + self.metrics['hardware_storage_capacity_bytes'].set(capacity, labels) + self._hw_set_health_metric(device.get('status', {}), hostname, device_id, 'storage') + + def _process_processors(self, status, hostname): + """Set CPU cores count and health metrics per processor.""" + for cpu_id, cpu in self._hw_iter_components(status, 'processors'): + cores = cpu.get('total_cores', 0) + labels = ( + hostname, + cpu_id, + cpu.get('manufacturer', 'unknown'), + cpu.get('model', 'unknown'), + cpu.get('total_threads', 'unknown') + ) + self.metrics['hardware_cpu_cores'].set(cores, labels) + self._hw_set_health_metric(cpu.get('status', {}), hostname, cpu_id, 'processors') + + def _process_memory(self, status, hostname): + """Set memory capacity (in bytes) and health metrics per DIMM.""" + for dimm_id, dimm in self._hw_iter_components(status, 'memory'): + capacity_mib = dimm.get('capacity_mi_b', 0) + capacity_bytes = capacity_mib * MIB_TO_BYTES + labels = ( + hostname, + dimm_id, + dimm.get('memory_device_type', 'unknown'), + ) + self.metrics['hardware_memory_capacity_bytes'].set(capacity_bytes, labels) + self._hw_set_health_metric(dimm.get('status', {}), hostname, dimm_id, 'memory') + + def _process_power_network(self, status, hostname): + """Set health metrics for power and network""" + for comp_id, comp in self._hw_iter_components(status, 'power'): + name = comp.get('name', comp_id) + self._hw_set_health_metric(comp.get('status', {}), hostname, name, 'power') + for comp_id, comp in self._hw_iter_components(status, 'network'): + self._hw_set_health_metric(comp.get('status', {}), hostname, comp_id, 'network') + + def _process_sensors(self, status, hostname): + """Set fan and temperature readings and health metrics""" + for category, sensor in SENSOR_METRICS.items(): + for comp_id, comp in self._hw_iter_components(status, category): + self._hw_set_sensor_metric(comp, comp_id, hostname, category, sensor['metric']) + + def _process_firmware(self, hostname, data): + """Set firmware info metrics (value=1) with version as label, skip unknown.""" + fw_data = data.get('firmware', data.get('firmwares', {})) + for fw_id, fw_info in fw_data.items(): + component = fw_info.get('name', fw_id) + version = fw_info.get('version', 'unknown') + if version and version != 'unknown': + labels = (hostname, component, str(version)) + self.metrics['hardware_firmware_info'].set(1, labels) + @profile_method(True) def get_hardware_metrics(self) -> None: + """Fetch node-proxy fullreport and export all hardware metrics.""" if not self.orch_is_available(): return try: report = raise_if_exception(self.node_proxy_fullreport()) - firmware_report = raise_if_exception(self.node_proxy_firmware()) except Exception as e: self.log.debug(f"node-proxy data not available: {e}") return - for host, data in report.items(): + for hostname, data in report.items(): status = data.get('status', {}) - - for sys_id, devices in status.get('storage', {}).items(): - for device_id, device in devices.items(): - capacity = device.get('capacity_bytes', 0) - labels = ( - host, - device_id, - device.get('model', 'unknown'), - device.get('protocol', 'unknown') - ) - self.metrics['node_proxy_storage_capacity_bytes'].set(capacity, labels) - - for sys_id, cpus in status.get('processors', {}).items(): - for cpu_id, cpu in cpus.items(): - cores = cpu.get('total_cores', 0) - labels = ( - host, - cpu_id, - cpu.get('manufacturer', 'unknown'), - cpu.get('model', 'unknown') - ) - self.metrics['node_proxy_cpu_cores'].set(cores, labels) - - for sys_id, dimms in status.get('memory', {}).items(): - for dimm_id, dimm in dimms.items(): - capacity_mib = dimm.get('capacity_mi_b', 0) - capacity_bytes = capacity_mib * MIB_TO_BYTES - labels = ( - host, - dimm_id, - dimm.get('memory_device_type', 'unknown'), - dimm.get('manufacturer', 'unknown') - ) - self.metrics['node_proxy_memory_capacity_bytes'].set(capacity_bytes, labels) - - for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network', 'temperatures']: - for sys_id, components in status.get(category, {}).items(): - for comp_id, comp in components.items(): - if not isinstance(comp, dict): - continue - status_val = comp.get('status', {}) - if isinstance(status_val, dict): - health_str = status_val.get('health', 'Unknown') - elif isinstance(status_val, str): - health_str = status_val - else: - health_str = 'Unknown' - - if health_str == 'OK': - health_value = 0 - elif health_str in ['Warning', 'Degraded']: - health_value = 1 - else: - health_value = 2 - - labels = (host, comp_id, category) - self.metrics['node_proxy_health'].set(health_value, labels) - - for sys_id, sensors in status.get('temperatures', {}).items(): - for sensor_id, sensor in sensors.items(): - reading = sensor.get('reading') - sensor_name = sensor.get('name', sensor_id) - if reading is None or reading == 'unknown': - continue - try: - temp_value = float(reading) - labels = (host, sensor_name) - self.metrics['node_proxy_temperature_celsius'].set(temp_value, labels) - except (ValueError, TypeError): - continue - - for sys_id, fans_data in status.get('fans', {}).items(): - for fan_id, fan in fans_data.items(): - rpm = fan.get('reading') - fan_name = fan.get('name', fan_id) - if rpm is None or rpm == 'unknown': - continue - try: - rpm_value = float(rpm) - labels = (host, fan_name) - self.metrics['node_proxy_fan_rpm'].set(rpm_value, labels) - except (ValueError, TypeError): - continue - - for host, fw_data in firmware_report.items(): - for fw_id, fw_info in fw_data.items(): - component = fw_info.get('name', fw_id) - version = fw_info.get('version', 'unknown') - if version and version != 'unknown': - labels = (host, component, str(version)) - self.metrics['node_proxy_firmware_info'].set(1, labels) + self._process_firmware(hostname, data) + self._process_storage(status, hostname) + self._process_processors(status, hostname) + self._process_memory(status, hostname) + self._process_power_network(status, hostname) + self._process_sensors(status, hostname) @profile_method(True) def collect(self) -> str: diff --git a/src/pybind/mgr/prometheus/test_module.py b/src/pybind/mgr/prometheus/test_module.py index 8a2a686de3e..fa844aa587c 100644 --- a/src/pybind/mgr/prometheus/test_module.py +++ b/src/pybind/mgr/prometheus/test_module.py @@ -181,3 +181,293 @@ class ThreadSafeLRUCacheDictTest(TestCase): self.assertEqual(len(cache), 100) for i in range(100): self.assertEqual(cache[f'key{i}'], f'value{i}') + + +MOCK_HW_FULLREPORT = { + 'at3n2.tuc.stglabs.ibm.com': { + 'host': 'at3n2.tuc.stglabs.ibm.com', + 'sn': '11S03NK990YM30ZP58E02X', + 'status': { + 'storage': {'Self': { + 'nvme_device0_nsid1': { + 'capacity_bytes': 512110190592, + 'model': 'Micron_2550_MTFDKBK512TGE', + 'protocol': 'NVMe', + 'serial_number': '24424BAA3C40', + 'firmware_version': 'V6MA001', + 'slot': '0', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}, + 'processors': {'Self': { + 'devtype1_cpu0': { + 'total_cores': 48, + 'total_threads': 96, + 'model': 'unknown', + 'manufacturer': 'Advanced Micro Devices, Inc.', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}, + 'memory': {'Self': { + 'devtype2_dimm0': { + 'memory_device_type': 'DDR5', + 'capacity_mi_b': 131072, + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}, + 'power': {'Self': { + '38': { + 'name': 'PSU1_PIN', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + '39': { + 'name': 'PSU1_POUT', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}, + 'network': {'Self': { + 'networkinterfaces_devtype7_nic0': { + 'name': 'NetworkAdapter_0', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}, + 'fans': {'Self': { + '0': { + 'name': 'FAN1_TACH_IN', + 'reading': 23940, + 'reading_units': 'RPM', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}, + 'temperatures': {'Self': { + '0': { + 'name': 'C1_DCSCM_TEMP', + 'reading': 32, + 'reading_units': 'Cel', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + '6': { + 'name': 'C1_CPU_TEMP', + 'reading': 47, + 'reading_units': 'Cel', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}, + }, + 'firmware': { + 'bios': { + 'name': 'BIOS', + 'version': 'unknown', + }, + 'bmc': { + 'name': 'BMC', + 'version': '0.50.d57cfd', + }, + 'cpld': { + 'name': 'CPLD', + 'version': 'unknown', + }, + }, + } +} + + +class HardwareMetricsTest(TestCase): + """Tests for hardware metrics processing from node-proxy fullreport data.""" + + def setUp(self): + from prometheus.module import ( + Module, HW_STORAGE_LABELS, HW_CPU_LABELS, HW_MEMORY_LABELS, + HW_HEALTH_LABELS, HW_TEMP_LABELS, HW_FAN_LABELS, HW_FIRMWARE_LABELS, + ) + self.module = mock.MagicMock(spec=Module) + self.module.log = mock.MagicMock() + self.module.metrics = { + 'hardware_storage_capacity_bytes': Metric( + 'gauge', 'hardware_storage_capacity_bytes', '', HW_STORAGE_LABELS), + 'hardware_cpu_cores': Metric( + 'gauge', 'hardware_cpu_cores', '', HW_CPU_LABELS), + 'hardware_memory_capacity_bytes': Metric( + 'gauge', 'hardware_memory_capacity_bytes', '', HW_MEMORY_LABELS), + 'hardware_health': Metric( + 'gauge', 'hardware_health', '', HW_HEALTH_LABELS), + 'hardware_temperature_celsius': Metric( + 'gauge', 'hardware_temperature_celsius', '', HW_TEMP_LABELS), + 'hardware_fan_rpm': Metric( + 'gauge', 'hardware_fan_rpm', '', HW_FAN_LABELS), + 'hardware_firmware_info': Metric( + 'gauge', 'hardware_firmware_info', '', HW_FIRMWARE_LABELS), + } + self.hostname = 'at3n2.tuc.stglabs.ibm.com' + self.data = MOCK_HW_FULLREPORT[self.hostname] + self.status = self.data['status'] + self.module._hw_get_health_value = Module._hw_get_health_value.__get__(self.module) + self.module._hw_set_health_metric = Module._hw_set_health_metric.__get__(self.module) + self.module._hw_iter_components = Module._hw_iter_components.__get__(self.module) + self.module._hw_set_sensor_metric = Module._hw_set_sensor_metric.__get__(self.module) + self.module._process_storage = Module._process_storage.__get__(self.module) + self.module._process_processors = Module._process_processors.__get__(self.module) + self.module._process_memory = Module._process_memory.__get__(self.module) + self.module._process_power_network = Module._process_power_network.__get__(self.module) + self.module._process_sensors = Module._process_sensors.__get__(self.module) + self.module._process_firmware = Module._process_firmware.__get__(self.module) + + # --- _hw_get_health_value --- + + def test_health_value_ok(self): + val = self.module._hw_get_health_value({'health': 'OK'}) + self.assertEqual(val, 0) + + def test_health_value_warning(self): + val = self.module._hw_get_health_value({'health': 'Warning'}) + self.assertEqual(val, 1) + + def test_health_value_critical(self): + val = self.module._hw_get_health_value({'health': 'Critical'}) + self.assertEqual(val, 2) + + def test_health_value_string_input(self): + val = self.module._hw_get_health_value('OK') + self.assertEqual(val, 0) + + def test_health_value_unknown_logs_warning(self): + val = self.module._hw_get_health_value( + {'health': 'Exploded'}, 'host1', 'comp1', 'storage') + self.assertIsNone(val) + self.module.log.warning.assert_called_once() + + def test_health_value_unknown_skips_metric(self): + self.module._hw_set_health_metric( + {'health': 'Exploded'}, 'host1', 'comp1', 'storage') + self.assertEqual(self.module.metrics['hardware_health'].value, {}) + + # --- _process_storage --- + + def test_storage_capacity_and_labels(self): + self.module._process_storage(self.status, self.hostname) + expected_labels = ( + self.hostname, 'nvme_device0_nsid1', + 'Micron_2550_MTFDKBK512TGE', 'NVMe', 'V6MA001', '0', '24424BAA3C40', + ) + self.assertEqual( + self.module.metrics['hardware_storage_capacity_bytes'].value[expected_labels], + 512110190592, + ) + + def test_storage_sets_health(self): + self.module._process_storage(self.status, self.hostname) + health_labels = (self.hostname, 'nvme_device0_nsid1', 'storage') + self.assertEqual( + self.module.metrics['hardware_health'].value[health_labels], 0) + + # --- _process_processors --- + + def test_cpu_cores_and_threads_label(self): + self.module._process_processors(self.status, self.hostname) + expected_labels = ( + self.hostname, 'devtype1_cpu0', + 'Advanced Micro Devices, Inc.', 'unknown', 96, + ) + self.assertEqual( + self.module.metrics['hardware_cpu_cores'].value[expected_labels], 48) + + # --- _process_memory --- + + def test_memory_mib_to_bytes_conversion(self): + self.module._process_memory(self.status, self.hostname) + expected_labels = (self.hostname, 'devtype2_dimm0', 'DDR5') + self.assertEqual( + self.module.metrics['hardware_memory_capacity_bytes'].value[expected_labels], + 131072 * 1048576, + ) + + # --- _process_power_network --- + + def test_power_uses_name_not_numeric_id(self): + """Power components use human-readable name (PSU1_PIN) not dict key (38).""" + self.module._process_power_network(self.status, self.hostname) + health = self.module.metrics['hardware_health'].value + self.assertIn((self.hostname, 'PSU1_PIN', 'power'), health) + self.assertIn((self.hostname, 'PSU1_POUT', 'power'), health) + self.assertNotIn((self.hostname, '38', 'power'), health) + self.assertNotIn((self.hostname, '39', 'power'), health) + + def test_network_uses_comp_id_not_generic_name(self): + """Network uses descriptive comp_id, not generic 'NetworkAdapter_0'.""" + self.module._process_power_network(self.status, self.hostname) + health = self.module.metrics['hardware_health'].value + self.assertIn( + (self.hostname, 'networkinterfaces_devtype7_nic0', 'network'), health) + self.assertNotIn( + (self.hostname, 'NetworkAdapter_0', 'network'), health) + + # --- _process_sensors --- + + def test_temperature_uses_sensor_name(self): + """Temperature reading uses 'C1_DCSCM_TEMP' not numeric key '0'.""" + self.module._process_sensors(self.status, self.hostname) + temp = self.module.metrics['hardware_temperature_celsius'].value + self.assertIn((self.hostname, 'C1_DCSCM_TEMP'), temp) + self.assertEqual(temp[(self.hostname, 'C1_DCSCM_TEMP')], 32.0) + self.assertNotIn((self.hostname, '0'), temp) + + def test_temperature_health_uses_sensor_name(self): + """Health metric for temperatures also uses sensor name, not numeric key.""" + self.module._process_sensors(self.status, self.hostname) + health = self.module.metrics['hardware_health'].value + self.assertIn((self.hostname, 'C1_DCSCM_TEMP', 'temperatures'), health) + self.assertIn((self.hostname, 'C1_CPU_TEMP', 'temperatures'), health) + self.assertNotIn((self.hostname, '0', 'temperatures'), health) + self.assertNotIn((self.hostname, '6', 'temperatures'), health) + + def test_fan_uses_fan_name(self): + """Fan reading uses 'FAN1_TACH_IN' not numeric key '0'.""" + self.module._process_sensors(self.status, self.hostname) + fan = self.module.metrics['hardware_fan_rpm'].value + self.assertIn((self.hostname, 'FAN1_TACH_IN'), fan) + self.assertEqual(fan[(self.hostname, 'FAN1_TACH_IN')], 23940.0) + + def test_sensor_unknown_reading_skipped(self): + """Sensors with 'unknown' reading should not set reading metric but still set health.""" + status = {'temperatures': {'Self': { + '99': { + 'name': 'GHOST_TEMP', + 'reading': 'unknown', + 'status': {'health': 'OK', 'state': 'Enabled'}, + }, + }}} + self.module._process_sensors(status, self.hostname) + self.assertNotIn( + (self.hostname, 'GHOST_TEMP'), + self.module.metrics['hardware_temperature_celsius'].value) + self.assertIn( + (self.hostname, 'GHOST_TEMP', 'temperatures'), + self.module.metrics['hardware_health'].value) + + # --- _process_firmware --- + + def test_firmware_known_version_exported(self): + self.module._process_firmware(self.hostname, self.data) + fw = self.module.metrics['hardware_firmware_info'].value + self.assertIn((self.hostname, 'BMC', '0.50.d57cfd'), fw) + self.assertEqual(fw[(self.hostname, 'BMC', '0.50.d57cfd')], 1) + + def test_firmware_unknown_version_skipped(self): + """BIOS and CPLD with version='unknown' should not be exported.""" + self.module._process_firmware(self.hostname, self.data) + fw = self.module.metrics['hardware_firmware_info'].value + for labels in fw: + self.assertNotEqual(labels[1], 'BIOS') + self.assertNotEqual(labels[1], 'CPLD') + + # --- label count consistency --- + + def test_storage_label_count_matches(self): + self.module._process_storage(self.status, self.hostname) + for labels in self.module.metrics['hardware_storage_capacity_bytes'].value: + self.assertEqual(len(labels), 7) + + def test_cpu_label_count_matches(self): + self.module._process_processors(self.status, self.hostname) + for labels in self.module.metrics['hardware_cpu_cores'].value: + self.assertEqual(len(labels), 5)