From: Afreen Misbah Date: Fri, 17 Jul 2026 15:51:40 +0000 (+0530) Subject: mgr/dashboard: fix stale metrics in hardware dashboards X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8321ca25d7cb20efd2afbeb83766f4119f64b47e;p=ceph.git mgr/dashboard: fix stale metrics in hardware dashboards Switch all stat panels in the hardware and hardware compression dashboards from lastNotNull to last reduceFunction. When a host is unreachable or node-proxy stops, metric silence is ambiguous — it could mean host down, agent crash, or scrape failure. lastNotNull freezes the last known value (e.g. 4 NVMe drives), causing inconsistency with the global Overview which reflects live count reductions. Use last so panels show N/A when no current data arrives. Fixes: https://tracker.ceph.com/issues/78360 Signed-off-by: Afreen Misbah --- diff --git a/monitoring/ceph-mixin/dashboards/hardware.libsonnet b/monitoring/ceph-mixin/dashboards/hardware.libsonnet index 74cd6e8162f..ea3d610cb71 100644 --- a/monitoring/ceph-mixin/dashboards/hardware.libsonnet +++ b/monitoring/ceph-mixin/dashboards/hardware.libsonnet @@ -78,7 +78,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='Health', unit='short', datasource='$datasource', - gridPosition={ h: 4, w: 4, x: 0, y: 1 } + gridPosition={ h: 4, w: 4, x: 0, y: 1 }, + reducerFunction='last', + noValue='N/A', ) .addTarget($.addTargetSchema( 'max(ceph_hardware_health)', @@ -92,6 +94,7 @@ local g = import 'grafonnet/grafana.libsonnet'; { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } }, { type: 'range', options: { from: 1, to: 999, result: { color: 'semi-dark-red', index: 1, text: 'NOT OK' } } }, ], + noValue: 'N/A', thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }], @@ -219,7 +222,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='Hosts', unit='short', datasource='$datasource', - gridPosition={ h: 4, w: 2, x: 0, y: 5 } + gridPosition={ h: 4, w: 2, x: 0, y: 5 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'count(count by(hostname) (ceph_hardware_health))', legendFormat='Hosts' @@ -231,7 +236,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='Drives', unit='short', datasource='$datasource', - gridPosition={ h: 4, w: 2, x: 2, y: 5 } + gridPosition={ h: 4, w: 2, x: 2, y: 5 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'count(ceph_hardware_storage_capacity_bytes)', legendFormat='Drives' @@ -266,7 +273,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='Power', unit='short', datasource='$datasource', - gridPosition={ h: 3, w: 2, x: 0, y: 2 } + gridPosition={ h: 3, w: 2, x: 0, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'max(ceph_hardware_health{hostname=~"$hostname",category="power"})', legendFormat='Power' @@ -279,6 +288,7 @@ local g = import 'grafonnet/grafana.libsonnet'; { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } }, { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } }, ], + noValue: 'N/A', thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] }, }, }, @@ -290,7 +300,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='MB Temperature (max)', unit='celsius', datasource='$datasource', - gridPosition={ h: 4, w: 3, x: 2, y: 2 } + gridPosition={ h: 4, w: 3, x: 2, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})', legendFormat='Motherboard' @@ -306,7 +318,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='CPU Temperature', unit='celsius', datasource='$datasource', - gridPosition={ h: 4, w: 3, x: 5, y: 2 } + gridPosition={ h: 4, w: 3, x: 5, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})', legendFormat='CPU' @@ -322,7 +336,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='DIMM Temperature (max)', unit='celsius', datasource='$datasource', - gridPosition={ h: 4, w: 3, x: 8, y: 2 } + gridPosition={ h: 4, w: 3, x: 8, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})', legendFormat='DIMM' @@ -338,7 +354,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='PSU Fans', unit='short', datasource='$datasource', - gridPosition={ h: 4, w: 3, x: 11, y: 2 } + gridPosition={ h: 4, w: 3, x: 11, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'count(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})', legendFormat='PSU Fans' @@ -353,7 +371,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='AVG PSU Temperature', unit='celsius', datasource='$datasource', - gridPosition={ h: 4, w: 3, x: 14, y: 2 } + gridPosition={ h: 4, w: 3, x: 14, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})', legendFormat='PSU' @@ -369,7 +389,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='NVMe Drives', unit='short', datasource='$datasource', - gridPosition={ h: 4, w: 3, x: 17, y: 2 } + gridPosition={ h: 4, w: 3, x: 17, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'count(ceph_hardware_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})', legendFormat='NVMe' @@ -385,7 +407,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='NVMe Temperature (max)', unit='celsius', datasource='$datasource', - gridPosition={ h: 4, w: 3, x: 20, y: 2 } + gridPosition={ h: 4, w: 3, x: 20, y: 2 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})', legendFormat='NVMe' @@ -401,7 +425,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='Network', unit='short', datasource='$datasource', - gridPosition={ h: 3, w: 2, x: 0, y: 5 } + gridPosition={ h: 3, w: 2, x: 0, y: 5 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'max(ceph_hardware_health{hostname=~"$hostname",category="network"})', legendFormat='Network' @@ -414,6 +440,7 @@ local g = import 'grafonnet/grafana.libsonnet'; { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } }, { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } }, ], + noValue: 'N/A', thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] }, }, }, @@ -425,7 +452,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='Cooling', unit='short', datasource='$datasource', - gridPosition={ h: 3, w: 2, x: 0, y: 8 } + gridPosition={ h: 3, w: 2, x: 0, y: 8 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'max(ceph_hardware_health{hostname=~"$hostname",category="fans"})', legendFormat='Cooling' @@ -438,6 +467,7 @@ local g = import 'grafonnet/grafana.libsonnet'; { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } }, { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } }, ], + noValue: 'N/A', thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] }, }, }, @@ -449,7 +479,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='Drives', unit='short', datasource='$datasource', - gridPosition={ h: 3, w: 2, x: 0, y: 11 } + gridPosition={ h: 3, w: 2, x: 0, y: 11 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'max(ceph_hardware_health{hostname=~"$hostname",category="storage"})', legendFormat='Drives' @@ -462,6 +494,7 @@ local g = import 'grafonnet/grafana.libsonnet'; { type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } }, { type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } }, ], + noValue: 'N/A', thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] }, }, }, @@ -559,7 +592,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='FAN: $fan_speeds (RPM)', unit='locale', datasource='$datasource', - gridPosition={ h: 5, w: 4, x: 0, y: 3 } + gridPosition={ h: 5, w: 4, x: 0, y: 3 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}', legendFormat='{{fan_name}}' @@ -577,7 +612,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='PSU1 Fan Speed (RPM)', unit='locale', datasource='$datasource', - gridPosition={ h: 5, w: 4, x: 0, y: 8 } + gridPosition={ h: 5, w: 4, x: 0, y: 8 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}', legendFormat='{{fan_name}}' @@ -592,7 +629,9 @@ local g = import 'grafonnet/grafana.libsonnet'; title='PSU2 Fan Speed (RPM)', unit='locale', datasource='$datasource', - gridPosition={ h: 5, w: 4, x: 4, y: 8 } + gridPosition={ h: 5, w: 4, x: 4, y: 8 }, + reducerFunction='last', + noValue='N/A', ).addTarget($.addTargetSchema( 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}', legendFormat='{{fan_name}}' diff --git a/monitoring/ceph-mixin/dashboards_out/ceph-hardware-compression.json b/monitoring/ceph-mixin/dashboards_out/ceph-hardware-compression.json index e116eccc246..7c1a5a51e26 100644 --- a/monitoring/ceph-mixin/dashboards_out/ceph-hardware-compression.json +++ b/monitoring/ceph-mixin/dashboards_out/ceph-hardware-compression.json @@ -109,7 +109,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "noValue": "N/A" }, "overrides": [] }, @@ -128,7 +129,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -193,7 +194,8 @@ } ] }, - "unit": "none" + "unit": "none", + "noValue": "N/A" }, "overrides": [] }, @@ -212,7 +214,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -269,7 +271,8 @@ } ] }, - "unit": "percent" + "unit": "percent", + "noValue": "N/A" }, "overrides": [] }, @@ -288,7 +291,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -351,7 +354,8 @@ } ] }, - "unit": "percent" + "unit": "percent", + "noValue": "N/A" }, "overrides": [] }, @@ -370,7 +374,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -422,7 +426,8 @@ "mode": "palette-classic" }, "mappings": [], - "unit": "bytes" + "unit": "bytes", + "noValue": "N/A" }, "overrides": [ { @@ -486,7 +491,7 @@ "pieType": "donut", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -549,7 +554,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "noValue": "N/A" }, "overrides": [] }, @@ -568,7 +574,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -616,7 +622,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "noValue": "N/A" }, "overrides": [] }, @@ -635,7 +642,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -683,7 +690,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "noValue": "N/A" }, "overrides": [] }, @@ -702,7 +710,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -750,7 +758,8 @@ } ] }, - "unit": "short" + "unit": "short", + "noValue": "N/A" }, "overrides": [] }, @@ -769,7 +778,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -829,7 +838,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "noValue": "N/A" }, "overrides": [] }, @@ -848,7 +858,7 @@ "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1515,4 +1525,4 @@ "uid": "ceph-hw-compression-v2", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/monitoring/ceph-mixin/dashboards_out/hardware.json b/monitoring/ceph-mixin/dashboards_out/hardware.json index fa16b7f9670..a5bcc887ab6 100644 --- a/monitoring/ceph-mixin/dashboards_out/hardware.json +++ b/monitoring/ceph-mixin/dashboards_out/hardware.json @@ -78,6 +78,7 @@ "type": "range" } ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -107,7 +108,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -415,6 +416,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ ] @@ -437,7 +439,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -467,6 +469,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ ] @@ -489,7 +492,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -623,6 +626,7 @@ "type": "range" } ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -652,7 +656,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -682,6 +686,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -716,7 +721,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -748,6 +753,7 @@ "mappings": [ ], "max": 85, "min": 0, + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -782,7 +788,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -814,6 +820,7 @@ "mappings": [ ], "max": 88, "min": 0, + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -848,7 +855,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -878,6 +885,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ ] @@ -900,7 +908,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -936,6 +944,7 @@ "mappings": [ ], "max": 100, "min": 0, + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -970,7 +979,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1004,6 +1013,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -1038,7 +1048,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1074,6 +1084,7 @@ "mappings": [ ], "max": 85, "min": 0, + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -1108,7 +1119,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1159,6 +1170,7 @@ "type": "range" } ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -1188,7 +1200,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1239,6 +1251,7 @@ "type": "range" } ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -1268,7 +1281,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1319,6 +1332,7 @@ "type": "range" } ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ @@ -1348,7 +1362,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1555,6 +1569,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ ] @@ -1578,7 +1593,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1614,6 +1629,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ ] @@ -1636,7 +1652,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -1670,6 +1686,7 @@ "decimals": 0, "links": [ ], "mappings": [ ], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ ] @@ -1692,7 +1709,7 @@ "orientation": "horizontal", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false