From: Afreen Misbah Date: Thu, 25 Jun 2026 08:18:19 +0000 (+0530) Subject: monitoring: fix hardware Grafana dashboard and health metrics X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=04207ca6dcfc86c51845912848d9cbf1a7cb87d0;p=ceph.git monitoring: fix hardware Grafana dashboard and health metrics - Fix fan repeating panels: set multi=true on fan_speeds template variable so Grafana generates one panel per fan instead of one - Remove TACH-only regex filter on fan_speeds template and AVG Cooling query so all system fans are visible regardless of naming - Replace duplicate Power Control panel with Network health panel - Fix NVMe drive count to use storage_capacity_bytes{protocol=NVMe} instead of counting temperature sensors (inaccurate proxy) - Normalize all hostname filters to regex match (=~) for consistency - Register hardware.libsonnet in dashboards.libsonnet so the dashboard JSON is generated during ceph-mixin builds - Add temperatures category to prometheus health metrics loop Signed-off-by: Afreen Misbah Assisted-by: Claude Signed-off-by: Afreen Misbah --- diff --git a/monitoring/ceph-mixin/dashboards.libsonnet b/monitoring/ceph-mixin/dashboards.libsonnet index ebd8c1bfc34..edb8078784b 100644 --- a/monitoring/ceph-mixin/dashboards.libsonnet +++ b/monitoring/ceph-mixin/dashboards.libsonnet @@ -12,5 +12,6 @@ (import 'dashboards/smb-overview.libsonnet') + (import 'dashboards/ceph-nvmeof.libsonnet') + (import 'dashboards/ceph-nvmeof-performance.libsonnet') + + (import 'dashboards/hardware.libsonnet') + { _config:: $._config }, } diff --git a/monitoring/ceph-mixin/dashboards/hardware.libsonnet b/monitoring/ceph-mixin/dashboards/hardware.libsonnet index 6d38bd2a441..f3886e008cb 100644 --- a/monitoring/ceph-mixin/dashboards/hardware.libsonnet +++ b/monitoring/ceph-mixin/dashboards/hardware.libsonnet @@ -59,14 +59,14 @@ local g = import 'grafonnet/grafana.libsonnet'; g.template.new( 'fan_speeds', '$datasource', - 'label_values(ceph_node_proxy_fan_rpm{hostname="$hostname"},fan_name)', + 'label_values(ceph_node_proxy_fan_rpm{hostname=~"$hostname"},fan_name)', label='', refresh='load', includeAll=true, - multi=false, + multi=true, allValues='', sort=0, - regex='/.*TACH.*/', + regex='', hide=2 ) ) @@ -275,7 +275,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 2 } ).addTarget($.addTargetSchema( - 'ceph_node_proxy_health{hostname="$hostname",category="power"}', + 'ceph_node_proxy_health{hostname=~"$hostname",category="power"}', legendFormat='__auto' )) + { @@ -353,7 +353,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 4, w: 3, x: 17, y: 2 } ).addTarget($.addTargetSchema( - 'count(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})', + 'count(ceph_node_proxy_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})', legendFormat='__auto' )), @@ -368,14 +368,14 @@ local g = import 'grafonnet/grafana.libsonnet'; legendFormat='__auto' )), - // Power Control + // Network $.addStatPanel( - title='Power Control', + title='Network', unit='short', datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 5 } ).addTarget($.addTargetSchema( - 'ceph_node_proxy_health{hostname="$hostname",category="power"}', + 'ceph_node_proxy_health{hostname=~"$hostname",category="network"}', legendFormat='__auto' )) + { @@ -398,7 +398,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 8 } ).addTarget($.addTargetSchema( - 'ceph_node_proxy_health{hostname="$hostname",category="fans"}', + 'ceph_node_proxy_health{hostname=~"$hostname",category="fans"}', legendFormat='__auto' )) + { @@ -421,7 +421,7 @@ local g = import 'grafonnet/grafana.libsonnet'; datasource='$datasource', gridPosition={ h: 3, w: 2, x: 0, y: 11 } ).addTarget($.addTargetSchema( - 'ceph_node_proxy_health{hostname="$hostname",category="storage"}', + 'ceph_node_proxy_health{hostname=~"$hostname",category="storage"}', legendFormat='__auto' )) + { @@ -591,7 +591,7 @@ local g = import 'grafonnet/grafana.libsonnet'; format='short', ).addTarget( g.prometheus.target( - 'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name=~".*TACH.*"})', + 'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name!~"PSU.*"})', legendFormat='System Fans' ) ) diff --git a/monitoring/ceph-mixin/dashboards_out/hardware.json b/monitoring/ceph-mixin/dashboards_out/hardware.json index 2691c71890d..94db72377a3 100644 --- a/monitoring/ceph-mixin/dashboards_out/hardware.json +++ b/monitoring/ceph-mixin/dashboards_out/hardware.json @@ -658,7 +658,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"power\"}", + "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"power\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -970,7 +970,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "count(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*\"})", + "expr": "count(ceph_node_proxy_storage_capacity_bytes{hostname=~\"$hostname\", protocol=\"NVMe\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -1102,14 +1102,14 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"power\"}", + "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"network\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", "refId": "A" } ], - "title": "Power Control", + "title": "Network", "transparent": false, "type": "stat" }, @@ -1182,7 +1182,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"fans\"}", + "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"fans\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -1262,7 +1262,7 @@ "pluginVersion": "9.1.3", "targets": [ { - "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"storage\"}", + "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"storage\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -1968,7 +1968,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name=~\".*TACH.*\"})", + "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name!~\"PSU.*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "System Fans", @@ -2091,12 +2091,12 @@ "hide": 2, "includeAll": true, "label": "", - "multi": false, + "multi": true, "name": "fan_speeds", "options": [ ], - "query": "label_values(ceph_node_proxy_fan_rpm{hostname=\"$hostname\"},fan_name)", + "query": "label_values(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\"},fan_name)", "refresh": 1, - "regex": "/.*TACH.*/", + "regex": "", "sort": 0, "tagValuesQuery": "", "tags": [ ], diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py index f2d8a230611..45b14fe6502 100644 --- a/src/pybind/mgr/prometheus/module.py +++ b/src/pybind/mgr/prometheus/module.py @@ -2111,7 +2111,7 @@ class Module(MgrModule, OrchestratorClientMixin): ) self.metrics['node_proxy_memory_capacity_mib'].set(capacity, labels) - for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network']: + for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network', 'temperatures']: for sys_id, components in status.get(category, {}).items(): for comp_id, comp in components.items(): health_str = comp.get('status', {}).get('health', 'Unknown')