monitoring: fix hardware Grafana dashboard and health metrics

- Fix fan repeating panels: set multi=true on fan_speeds template
  variable so Grafana generates one panel per fan instead of one
- Remove TACH-only regex filter on fan_speeds template and AVG
  Cooling query so all system fans are visible regardless of naming
- Replace duplicate Power Control panel with Network health panel
- Fix NVMe drive count to use storage_capacity_bytes{protocol=NVMe}
  instead of counting temperature sensors (inaccurate proxy)
- Normalize all hostname filters to regex match (=~) for consistency
- Register hardware.libsonnet in dashboards.libsonnet so the
  dashboard JSON is generated during ceph-mixin builds
- Add temperatures category to prometheus health metrics loop

Signed-off-by: Afreen Misbah <afreen@ibm.com>
Assisted-by: Claude
Signed-off-by: Afreen Misbah <afreen@ibm.com>

diff --git a/monitoring/ceph-mixin/dashboards.libsonnet b/monitoring/ceph-mixin/dashboards.libsonnet
index ebd8c1bfc349b306b6810ed22cb8027430c88daf..edb8078784b8d431e5e1b5ed42126b4adbbb8b0b 100644 (file)
--- a/monitoring/ceph-mixin/dashboards.libsonnet
+++ b/monitoring/ceph-mixin/dashboards.libsonnet
@@ -12,5 +12,6 @@
     (import 'dashboards/smb-overview.libsonnet') +
     (import 'dashboards/ceph-nvmeof.libsonnet') +
     (import 'dashboards/ceph-nvmeof-performance.libsonnet') +
+    (import 'dashboards/hardware.libsonnet') +
     { _config:: $._config },
 }

diff --git a/monitoring/ceph-mixin/dashboards/hardware.libsonnet b/monitoring/ceph-mixin/dashboards/hardware.libsonnet
index 6d38bd2a4415dd92b1da468aa92b112971801c8d..f3886e008cb5ab0132876b610a08417171dc1194 100644 (file)
--- a/monitoring/ceph-mixin/dashboards/hardware.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/hardware.libsonnet
@@ -59,14 +59,14 @@ local g = import 'grafonnet/grafana.libsonnet';
       g.template.new(
         'fan_speeds',
         '$datasource',
-        'label_values(ceph_node_proxy_fan_rpm{hostname="$hostname"},fan_name)',
+        'label_values(ceph_node_proxy_fan_rpm{hostname=~"$hostname"},fan_name)',
         label='',
         refresh='load',
         includeAll=true,
-        multi=false,
+        multi=true,
         allValues='',
         sort=0,
-        regex='/.*TACH.*/',
+        regex='',
         hide=2
       )
     )
@@ -275,7 +275,7 @@ local g = import 'grafonnet/grafana.libsonnet';
           datasource='$datasource',
           gridPosition={ h: 3, w: 2, x: 0, y: 2 }
         ).addTarget($.addTargetSchema(
-          'ceph_node_proxy_health{hostname="$hostname",category="power"}',
+          'ceph_node_proxy_health{hostname=~"$hostname",category="power"}',
           legendFormat='__auto'
         ))
         + {
@@ -353,7 +353,7 @@ local g = import 'grafonnet/grafana.libsonnet';
           datasource='$datasource',
           gridPosition={ h: 4, w: 3, x: 17, y: 2 }
         ).addTarget($.addTargetSchema(
-          'count(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})',
+          'count(ceph_node_proxy_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})',
           legendFormat='__auto'
         )),
 
@@ -368,14 +368,14 @@ local g = import 'grafonnet/grafana.libsonnet';
           legendFormat='__auto'
         )),
 
-        // Power Control
+        // Network
         $.addStatPanel(
-          title='Power Control',
+          title='Network',
           unit='short',
           datasource='$datasource',
           gridPosition={ h: 3, w: 2, x: 0, y: 5 }
         ).addTarget($.addTargetSchema(
-          'ceph_node_proxy_health{hostname="$hostname",category="power"}',
+          'ceph_node_proxy_health{hostname=~"$hostname",category="network"}',
           legendFormat='__auto'
         ))
         + {
@@ -398,7 +398,7 @@ local g = import 'grafonnet/grafana.libsonnet';
           datasource='$datasource',
           gridPosition={ h: 3, w: 2, x: 0, y: 8 }
         ).addTarget($.addTargetSchema(
-          'ceph_node_proxy_health{hostname="$hostname",category="fans"}',
+          'ceph_node_proxy_health{hostname=~"$hostname",category="fans"}',
           legendFormat='__auto'
         ))
         + {
@@ -421,7 +421,7 @@ local g = import 'grafonnet/grafana.libsonnet';
           datasource='$datasource',
           gridPosition={ h: 3, w: 2, x: 0, y: 11 }
         ).addTarget($.addTargetSchema(
-          'ceph_node_proxy_health{hostname="$hostname",category="storage"}',
+          'ceph_node_proxy_health{hostname=~"$hostname",category="storage"}',
           legendFormat='__auto'
         ))
         + {
@@ -591,7 +591,7 @@ local g = import 'grafonnet/grafana.libsonnet';
           format='short',
         ).addTarget(
           g.prometheus.target(
-            'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name=~".*TACH.*"})',
+            'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name!~"PSU.*"})',
             legendFormat='System Fans'
           )
         )

diff --git a/monitoring/ceph-mixin/dashboards_out/hardware.json b/monitoring/ceph-mixin/dashboards_out/hardware.json
index 2691c71890d3d2645cbaf65ab2487dd6f0067512..94db72377a3049b8832af3fb4d672bc96bc15881 100644 (file)
--- a/monitoring/ceph-mixin/dashboards_out/hardware.json
+++ b/monitoring/ceph-mixin/dashboards_out/hardware.json
@@ -658,7 +658,7 @@
                "pluginVersion": "9.1.3",
                "targets": [
                   {
-                     "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"power\"}",
+                     "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"power\"}",
                      "format": "time_series",
                      "intervalFactor": 1,
                      "legendFormat": "__auto",
@@ -970,7 +970,7 @@
                "pluginVersion": "9.1.3",
                "targets": [
                   {
-                     "expr": "count(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*\"})",
+                     "expr": "count(ceph_node_proxy_storage_capacity_bytes{hostname=~\"$hostname\", protocol=\"NVMe\"})",
                      "format": "time_series",
                      "intervalFactor": 1,
                      "legendFormat": "__auto",
@@ -1102,14 +1102,14 @@
                "pluginVersion": "9.1.3",
                "targets": [
                   {
-                     "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"power\"}",
+                     "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"network\"}",
                      "format": "time_series",
                      "intervalFactor": 1,
                      "legendFormat": "__auto",
                      "refId": "A"
                   }
                ],
-               "title": "Power Control",
+               "title": "Network",
                "transparent": false,
                "type": "stat"
             },
@@ -1182,7 +1182,7 @@
                "pluginVersion": "9.1.3",
                "targets": [
                   {
-                     "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"fans\"}",
+                     "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"fans\"}",
                      "format": "time_series",
                      "intervalFactor": 1,
                      "legendFormat": "__auto",
@@ -1262,7 +1262,7 @@
                "pluginVersion": "9.1.3",
                "targets": [
                   {
-                     "expr": "ceph_node_proxy_health{hostname=\"$hostname\",category=\"storage\"}",
+                     "expr": "ceph_node_proxy_health{hostname=~\"$hostname\",category=\"storage\"}",
                      "format": "time_series",
                      "intervalFactor": 1,
                      "legendFormat": "__auto",
@@ -1968,7 +1968,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name=~\".*TACH.*\"})",
+                     "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name!~\"PSU.*\"})",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "System Fans",
@@ -2091,12 +2091,12 @@
             "hide": 2,
             "includeAll": true,
             "label": "",
-            "multi": false,
+            "multi": true,
             "name": "fan_speeds",
             "options": [ ],
-            "query": "label_values(ceph_node_proxy_fan_rpm{hostname=\"$hostname\"},fan_name)",
+            "query": "label_values(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\"},fan_name)",
             "refresh": 1,
-            "regex": "/.*TACH.*/",
+            "regex": "",
             "sort": 0,
             "tagValuesQuery": "",
             "tags": [ ],

diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py
index f2d8a2306110913408f6cd1920af37d083bb13be..45b14fe6502e9816d43008841a02caa22d4ccfc2 100644 (file)
--- a/src/pybind/mgr/prometheus/module.py
+++ b/src/pybind/mgr/prometheus/module.py
@@ -2111,7 +2111,7 @@ class Module(MgrModule, OrchestratorClientMixin):
                     )
                     self.metrics['node_proxy_memory_capacity_mib'].set(capacity, labels)
 
-            for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network']:
+            for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network', 'temperatures']:
                 for sys_id, components in status.get(category, {}).items():
                     for comp_id, comp in components.items():
                         health_str = comp.get('status', {}).get('health', 'Unknown')