]> git-server-git.apps.pok.os.sepia.ceph.com Git - ceph.git/commitdiff
mgr/dashboard: fix stale metrics in hardware dashboards 70292/head
authorAfreen Misbah <afreen@ibm.com>
Fri, 17 Jul 2026 15:51:40 +0000 (21:21 +0530)
committerAfreen Misbah <afreen@ibm.com>
Wed, 22 Jul 2026 20:13:38 +0000 (01:43 +0530)
Switch all stat panels in the hardware and hardware compression
dashboards from lastNotNull to last reduceFunction. When a host is
unreachable or node-proxy stops, metric silence is ambiguous — it
could mean host down, agent crash, or scrape failure. lastNotNull
freezes the last known value (e.g. 4 NVMe drives), causing
inconsistency with the global Overview which reflects live count
reductions.

Use last so panels show N/A when no current data arrives.

Fixes: https://tracker.ceph.com/issues/78360
Signed-off-by: Afreen Misbah <afreen@ibm.com>
monitoring/ceph-mixin/dashboards/hardware.libsonnet
monitoring/ceph-mixin/dashboards_out/ceph-hardware-compression.json
monitoring/ceph-mixin/dashboards_out/hardware.json

index 74cd6e8162fdbbf298a4edc2aae36f943572f3a1..ea3d610cb71fd208eef1c307445a454ad45fc57d 100644 (file)
@@ -78,7 +78,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='Health',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 4, x: 0, y: 1 }
+          gridPosition={ h: 4, w: 4, x: 0, y: 1 },
+          reducerFunction='last',
+          noValue='N/A',
         )
         .addTarget($.addTargetSchema(
           'max(ceph_hardware_health)',
@@ -92,6 +94,7 @@ local g = import 'grafonnet/grafana.libsonnet';
                 { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
                 { type: 'range', options: { from: 1, to: 999, result: { color: 'semi-dark-red', index: 1, text: 'NOT OK' } } },
               ],
+              noValue: 'N/A',
               thresholds: {
                 mode: 'absolute',
                 steps: [{ color: 'green' }, { color: 'red', value: 1 }],
@@ -219,7 +222,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='Hosts',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 2, x: 0, y: 5 }
+          gridPosition={ h: 4, w: 2, x: 0, y: 5 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'count(count by(hostname) (ceph_hardware_health))',
           legendFormat='Hosts'
@@ -231,7 +236,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='Drives',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 2, x: 2, y: 5 }
+          gridPosition={ h: 4, w: 2, x: 2, y: 5 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'count(ceph_hardware_storage_capacity_bytes)',
           legendFormat='Drives'
@@ -266,7 +273,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='Power',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 3, w: 2, x: 0, y: 2 }
+          gridPosition={ h: 3, w: 2, x: 0, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'max(ceph_hardware_health{hostname=~"$hostname",category="power"})',
           legendFormat='Power'
@@ -279,6 +288,7 @@ local g = import 'grafonnet/grafana.libsonnet';
                 { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
                 { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
               ],
+              noValue: 'N/A',
               thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
             },
           },
@@ -290,7 +300,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='MB Temperature (max)',
           unit='celsius',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 3, x: 2, y: 2 }
+          gridPosition={ h: 4, w: 3, x: 2, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})',
           legendFormat='Motherboard'
@@ -306,7 +318,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='CPU Temperature',
           unit='celsius',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 3, x: 5, y: 2 }
+          gridPosition={ h: 4, w: 3, x: 5, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})',
           legendFormat='CPU'
@@ -322,7 +336,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='DIMM Temperature (max)',
           unit='celsius',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 3, x: 8, y: 2 }
+          gridPosition={ h: 4, w: 3, x: 8, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})',
           legendFormat='DIMM'
@@ -338,7 +354,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='PSU Fans',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 3, x: 11, y: 2 }
+          gridPosition={ h: 4, w: 3, x: 11, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'count(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
           legendFormat='PSU Fans'
@@ -353,7 +371,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='AVG PSU Temperature',
           unit='celsius',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 3, x: 14, y: 2 }
+          gridPosition={ h: 4, w: 3, x: 14, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})',
           legendFormat='PSU'
@@ -369,7 +389,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='NVMe Drives',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 3, x: 17, y: 2 }
+          gridPosition={ h: 4, w: 3, x: 17, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'count(ceph_hardware_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})',
           legendFormat='NVMe'
@@ -385,7 +407,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='NVMe Temperature (max)',
           unit='celsius',
           datasource='$datasource',
-          gridPosition={ h: 4, w: 3, x: 20, y: 2 }
+          gridPosition={ h: 4, w: 3, x: 20, y: 2 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})',
           legendFormat='NVMe'
@@ -401,7 +425,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='Network',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 3, w: 2, x: 0, y: 5 }
+          gridPosition={ h: 3, w: 2, x: 0, y: 5 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'max(ceph_hardware_health{hostname=~"$hostname",category="network"})',
           legendFormat='Network'
@@ -414,6 +440,7 @@ local g = import 'grafonnet/grafana.libsonnet';
                 { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
                 { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
               ],
+              noValue: 'N/A',
               thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
             },
           },
@@ -425,7 +452,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='Cooling',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 3, w: 2, x: 0, y: 8 }
+          gridPosition={ h: 3, w: 2, x: 0, y: 8 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'max(ceph_hardware_health{hostname=~"$hostname",category="fans"})',
           legendFormat='Cooling'
@@ -438,6 +467,7 @@ local g = import 'grafonnet/grafana.libsonnet';
                 { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
                 { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
               ],
+              noValue: 'N/A',
               thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
             },
           },
@@ -449,7 +479,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='Drives',
           unit='short',
           datasource='$datasource',
-          gridPosition={ h: 3, w: 2, x: 0, y: 11 }
+          gridPosition={ h: 3, w: 2, x: 0, y: 11 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'max(ceph_hardware_health{hostname=~"$hostname",category="storage"})',
           legendFormat='Drives'
@@ -462,6 +494,7 @@ local g = import 'grafonnet/grafana.libsonnet';
                 { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
                 { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
               ],
+              noValue: 'N/A',
               thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
             },
           },
@@ -559,7 +592,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='FAN: $fan_speeds (RPM)',
           unit='locale',
           datasource='$datasource',
-          gridPosition={ h: 5, w: 4, x: 0, y: 3 }
+          gridPosition={ h: 5, w: 4, x: 0, y: 3 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}',
           legendFormat='{{fan_name}}'
@@ -577,7 +612,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='PSU1 Fan Speed (RPM)',
           unit='locale',
           datasource='$datasource',
-          gridPosition={ h: 5, w: 4, x: 0, y: 8 }
+          gridPosition={ h: 5, w: 4, x: 0, y: 8 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}',
           legendFormat='{{fan_name}}'
@@ -592,7 +629,9 @@ local g = import 'grafonnet/grafana.libsonnet';
           title='PSU2 Fan Speed (RPM)',
           unit='locale',
           datasource='$datasource',
-          gridPosition={ h: 5, w: 4, x: 4, y: 8 }
+          gridPosition={ h: 5, w: 4, x: 4, y: 8 },
+          reducerFunction='last',
+          noValue='N/A',
         ).addTarget($.addTargetSchema(
           'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}',
           legendFormat='{{fan_name}}'
index e116eccc246fff8cf8221f7839d5478cfa873ce9..7c1a5a51e26770b38631e36098d11f904541bbca 100644 (file)
               }
             ]
           },
-          "unit": "bytes"
+          "unit": "bytes",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "none"
+          "unit": "none",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "percent"
+          "unit": "percent",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "percent"
+          "unit": "percent",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
             "mode": "palette-classic"
           },
           "mappings": [],
-          "unit": "bytes"
+          "unit": "bytes",
+          "noValue": "N/A"
         },
         "overrides": [
           {
         "pieType": "donut",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "bytes"
+          "unit": "bytes",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "bytes"
+          "unit": "bytes",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "bytes"
+          "unit": "bytes",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "short"
+          "unit": "short",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
               }
             ]
           },
-          "unit": "bytes"
+          "unit": "bytes",
+          "noValue": "N/A"
         },
         "overrides": []
       },
         "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
           "fields": "",
           "values": false
   "uid": "ceph-hw-compression-v2",
   "version": 1,
   "weekStart": ""
-}
\ No newline at end of file
+}
index fa16b7f9670c55e7414de61b305018af329f4ce4..a5bcc887ab683a9c83d975db380017ff908ab7fa 100644 (file)
@@ -78,6 +78,7 @@
                            "type": "range"
                         }
                      ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [ ]
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [ ]
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                            "type": "range"
                         }
                      ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "mappings": [ ],
                      "max": 85,
                      "min": 0,
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "mappings": [ ],
                      "max": 88,
                      "min": 0,
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [ ]
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "mappings": [ ],
                      "max": 100,
                      "min": 0,
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "mappings": [ ],
                      "max": 85,
                      "min": 0,
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                            "type": "range"
                         }
                      ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                            "type": "range"
                         }
                      ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                            "type": "range"
                         }
                      ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [ ]
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [ ]
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false
                      "decimals": 0,
                      "links": [ ],
                      "mappings": [ ],
+                     "noValue": "N/A",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [ ]
                   "orientation": "horizontal",
                   "reduceOptions": {
                      "calcs": [
-                        "lastNotNull"
+                        "last"
                      ],
                      "fields": "",
                      "values": false