- export `firmware_version` labels from `node_proxy_storage_capacity_bytes` metrics to be used in device firmware panel
- improve iterations for performance - dropping redundant node_proxy_firmware() RPC; firmware data is already
present in the fullreport response, and removing unnecessary loops.
- replace health if/elif chain with HEALTH_STATUS_MAP dict lookup
- rename metrics to ceph_hardware_*, fix component labels and add tests
- added unit test cases
- add serial name, slot info to capacity metric
- fix health metrics showing ID instead of component name
Signed-off-by: Afreen Misbah <afreen@ibm.com>
$.addTemplateSchema(
'hostname',
'$datasource',
- 'label_values(ceph_node_proxy_health, hostname)',
+ 'label_values(ceph_hardware_health, hostname)',
1,
false,
1,
g.template.new(
'fan_speeds',
'$datasource',
- 'label_values(ceph_node_proxy_fan_rpm{hostname=~"$hostname"},fan_name)',
+ 'label_values(ceph_hardware_fan_rpm{hostname=~"$hostname"},fan_name)',
label='',
refresh='load',
includeAll=true,
multi=true,
allValues='',
sort=0,
- regex='',
+ regex='/.*TACH_OUT/',
hide=2
)
)
gridPosition={ h: 4, w: 4, x: 0, y: 1 }
)
.addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_health)',
+ 'max(ceph_hardware_health)',
legendFormat='__auto'
))
+ {
{ color: 'red', value: 85 },
])
.addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_temperature_celsius{sensor_name=~".*CPU_TEMP"})',
+ 'max(ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"})',
legendFormat='__auto'
)),
gridPos={ x: 7, y: 1, w: 3, h: 8 }
)
.addTarget($.addTargetSchema(
- 'count by(version) (ceph_node_proxy_firmware_info{component=~".*BMC.*"})',
+ 'count by(version) (ceph_hardware_firmware_info{component=~".*BMC.*"})',
legendFormat='{{version}}'
))
+ {
gridPos={ x: 10, y: 1, w: 3, h: 8 }
)
.addTarget($.addTargetSchema(
- 'count by(version) (ceph_node_proxy_firmware_info{component=~".*BIOS.*"})',
+ 'count by(version) (ceph_hardware_firmware_info{component=~".*BIOS.*"})',
legendFormat='{{version}}'
))
+ {
gridPos={ x: 13, y: 1, w: 6, h: 8 }
)
.addTarget($.addTargetSchema(
- 'count by(model) (ceph_node_proxy_storage_capacity_bytes)',
+ 'count by(model) (ceph_hardware_storage_capacity_bytes)',
legendFormat='{{model}}'
))
+ {
gridPos={ x: 19, y: 1, w: 5, h: 8 }
)
.addTarget($.addTargetSchema(
- 'count by(version) (ceph_node_proxy_firmware_info{component=~".*Drive.*"})',
- legendFormat='{{version}}',
+ 'count by(firmware_version) (ceph_hardware_storage_capacity_bytes{firmware_version!="unknown"})',
+ legendFormat='{{firmware_version}}',
instant=true
))
+ {
datasource='$datasource',
gridPosition={ h: 4, w: 2, x: 0, y: 5 }
).addTarget($.addTargetSchema(
- 'count(count by(hostname) (ceph_node_proxy_health))',
+ 'count(count by(hostname) (ceph_hardware_health))',
legendFormat='__auto'
)),
datasource='$datasource',
gridPosition={ h: 4, w: 2, x: 2, y: 5 }
).addTarget($.addTargetSchema(
- 'count(ceph_node_proxy_storage_capacity_bytes)',
+ 'count(ceph_hardware_storage_capacity_bytes)',
legendFormat='__auto'
)),
{ color: 'red', value: 80 },
])
.addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_temperature_celsius{sensor_name=~"NVME.*_TEMP"})',
+ 'max(ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"})',
legendFormat='__auto'
)),
] },
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 2 }
).addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_health{hostname=~"$hostname",category="power"})',
+ 'max(ceph_hardware_health{hostname=~"$hostname",category="power"})',
legendFormat='__auto'
))
+ {
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 2, y: 2 }
).addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})',
+ 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})',
legendFormat='__auto'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'red', value: 80 }] } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
+ },
// CPU Temperature
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 5, y: 2 }
).addTarget($.addTargetSchema(
- 'avg(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})',
+ 'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})',
legendFormat='__auto'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { max: 85, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 80 }, { color: 'semi-dark-red', value: 90 }] } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
+ },
// DIMM Temperature
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 8, y: 2 }
).addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})',
+ 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})',
legendFormat='__auto'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { max: 88, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'yellow', value: 70 }, { color: 'semi-dark-red', value: 80 }] } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
+ },
// PSU Fans
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 11, y: 2 }
).addTarget($.addTargetSchema(
- 'count(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
+ 'count(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
legendFormat='__auto'
- )),
+ ))
+ + { options+: { colorMode: 'none', graphMode: 'none' } },
// AVG PSU Temperature
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 14, y: 2 }
).addTarget($.addTargetSchema(
- 'avg(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})',
+ 'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})',
legendFormat='__auto'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' }, max: 100, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'yellow', value: 80 }, { color: 'semi-dark-red', value: 90 }] } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
+ },
// NVMe Drives
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 17, y: 2 }
).addTarget($.addTargetSchema(
- 'count(ceph_node_proxy_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})',
+ 'count(ceph_hardware_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})',
legendFormat='__auto'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'semi-dark-red', value: 78 }] } } },
+ options+: { colorMode: 'none', graphMode: 'none' },
+ },
// NVMe Temperature
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 20, y: 2 }
).addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})',
+ 'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})',
legendFormat='__auto'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'thresholds' }, max: 85, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'semi-dark-red', value: 78 }] } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
+ },
// Network
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 5 }
).addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_health{hostname=~"$hostname",category="network"})',
+ 'max(ceph_hardware_health{hostname=~"$hostname",category="network"})',
legendFormat='__auto'
))
+ {
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 8 }
).addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_health{hostname=~"$hostname",category="fans"})',
+ 'max(ceph_hardware_health{hostname=~"$hostname",category="fans"})',
legendFormat='__auto'
))
+ {
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 11 }
).addTarget($.addTargetSchema(
- 'max(ceph_node_proxy_health{hostname=~"$hostname",category="storage"})',
+ 'max(ceph_hardware_health{hostname=~"$hostname",category="storage"})',
legendFormat='__auto'
))
+ {
},
)
.addTarget($.addTargetSchema(
- 'ceph_node_proxy_storage_capacity_bytes{hostname=~"$hostname"}',
+ 'ceph_hardware_storage_capacity_bytes{hostname=~"$hostname"}',
legendFormat='__auto',
instant=true
) + { format: 'table' })
},
)
.addTarget($.addTargetSchema(
- 'ceph_node_proxy_firmware_info{hostname=~"$hostname"}',
+ 'ceph_hardware_firmware_info{hostname=~"$hostname"}',
legendFormat='__auto',
instant=true
) + { format: 'table' })
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 0, y: 3 }
).addTarget($.addTargetSchema(
- 'ceph_node_proxy_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}',
+ 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}',
legendFormat='{{fan_name}}'
))
+ {
+ fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
repeat: 'fan_speeds',
repeatDirection: 'h',
maxPerRow: 6,
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 0, y: 8 }
).addTarget($.addTargetSchema(
- 'ceph_node_proxy_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}',
+ 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}',
legendFormat='{{fan_name}}'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
+ },
// PSU2
$.addStatPanel(
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 4, y: 8 }
).addTarget($.addTargetSchema(
- 'ceph_node_proxy_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}',
+ 'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}',
legendFormat='{{fan_name}}'
- )),
+ ))
+ + {
+ fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } },
+ options+: { colorMode: 'none', graphMode: 'area' },
+ },
] },
// Row 4: Temperature History
format='celsius',
).addTarget(
g.prometheus.target(
- 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"}',
+ 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
format='celsius',
).addTarget(
g.prometheus.target(
- 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"}',
+ 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
format='celsius',
).addTarget(
g.prometheus.target(
- 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"}',
+ 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
format='celsius',
).addTarget(
g.prometheus.target(
- 'ceph_node_proxy_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*_TEMP"}',
+ 'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*_TEMP"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
g.graphPanel.new(
title='AVG PSU Fan Speed',
datasource='$datasource',
- format='none',
+ format='locale',
+ fill=1,
).addTarget(
g.prometheus.target(
- 'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
+ 'avg(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
legendFormat='PSU Fans'
)
)
description: 'AVG across both PSUs',
},
- // AVG Cooling Fan Speed
+ // AVG Cooling Fan RPMs
g.graphPanel.new(
- title='AVG Cooling Fan Speed',
+ title='AVG Cooling Fan RPMs',
datasource='$datasource',
- format='none',
+ format='locale',
+ fill=1,
).addTarget(
g.prometheus.target(
- 'avg(ceph_node_proxy_fan_rpm{hostname=~"$hostname", fan_name!~"PSU.*"})',
+ 'avg(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name!~"PSU.*"})',
legendFormat='System Fans'
)
)
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_health)",
+ "expr": "max(ceph_hardware_health)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_temperature_celsius{sensor_name=~\".*CPU_TEMP\"})",
+ "expr": "max(ceph_hardware_temperature_celsius{sensor_name=~\".*CPU_TEMP\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
},
"targets": [
{
- "expr": "count by(version) (ceph_node_proxy_firmware_info{component=~\".*BMC.*\"})",
+ "expr": "count by(version) (ceph_hardware_firmware_info{component=~\".*BMC.*\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{version}}",
},
"targets": [
{
- "expr": "count by(version) (ceph_node_proxy_firmware_info{component=~\".*BIOS.*\"})",
+ "expr": "count by(version) (ceph_hardware_firmware_info{component=~\".*BIOS.*\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{version}}",
},
"targets": [
{
- "expr": "count by(model) (ceph_node_proxy_storage_capacity_bytes)",
+ "expr": "count by(model) (ceph_hardware_storage_capacity_bytes)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{model}}",
},
"targets": [
{
- "expr": "count by(version) (ceph_node_proxy_firmware_info{component=~\".*Drive.*\"})",
+ "expr": "count by(firmware_version) (ceph_hardware_storage_capacity_bytes{firmware_version!=\"unknown\"})",
"format": "time_series",
"instant": true,
"intervalFactor": 1,
- "legendFormat": "{{version}}",
+ "legendFormat": "{{firmware_version}}",
"refId": "A"
}
],
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "count(count by(hostname) (ceph_node_proxy_health))",
+ "expr": "count(count by(hostname) (ceph_hardware_health))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "count(ceph_node_proxy_storage_capacity_bytes)",
+ "expr": "count(ceph_hardware_storage_capacity_bytes)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_temperature_celsius{sensor_name=~\"NVME.*_TEMP\"})",
+ "expr": "max(ceph_hardware_temperature_celsius{sensor_name=~\"NVME.*_TEMP\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"power\"})",
+ "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"power\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"mappings": [ ],
"thresholds": {
"mode": "absolute",
- "steps": [ ]
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "#EAB839",
+ "value": 70
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
},
"unit": "celsius"
}
"links": [ ],
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"})",
+ "expr": "max(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"decimals": 0,
"links": [ ],
"mappings": [ ],
+ "max": 85,
+ "min": 0,
"thresholds": {
"mode": "absolute",
- "steps": [ ]
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "#EAB839",
+ "value": 80
+ },
+ {
+ "color": "semi-dark-red",
+ "value": 90
+ }
+ ]
},
"unit": "celsius"
}
"links": [ ],
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "avg(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"})",
+ "expr": "avg(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"decimals": 0,
"links": [ ],
"mappings": [ ],
+ "max": 88,
+ "min": 0,
"thresholds": {
"mode": "absolute",
- "steps": [ ]
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "yellow",
+ "value": 70
+ },
+ {
+ "color": "semi-dark-red",
+ "value": 80
+ }
+ ]
},
"unit": "celsius"
}
"links": [ ],
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"})",
+ "expr": "max(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "count(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})",
+ "expr": "count(ceph_hardware_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"description": "",
"fieldConfig": {
"defaults": {
+ "color": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ },
"decimals": 0,
"links": [ ],
"mappings": [ ],
+ "max": 100,
+ "min": 0,
"thresholds": {
"mode": "absolute",
- "steps": [ ]
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "yellow",
+ "value": 80
+ },
+ {
+ "color": "semi-dark-red",
+ "value": 90
+ }
+ ]
},
"unit": "celsius"
}
"links": [ ],
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "avg(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"PSU.*_TEMP.*\"})",
+ "expr": "avg(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"PSU.*_TEMP.*\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"description": "",
"fieldConfig": {
"defaults": {
+ "color": {
+ "fixedColor": "blue",
+ "mode": "thresholds"
+ },
"decimals": 0,
"links": [ ],
"mappings": [ ],
"thresholds": {
"mode": "absolute",
- "steps": [ ]
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "#EAB839",
+ "value": 70
+ },
+ {
+ "color": "semi-dark-red",
+ "value": 78
+ }
+ ]
},
"unit": "short"
}
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "count(ceph_node_proxy_storage_capacity_bytes{hostname=~\"$hostname\", protocol=\"NVMe\"})",
+ "expr": "count(ceph_hardware_storage_capacity_bytes{hostname=~\"$hostname\", protocol=\"NVMe\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"description": "",
"fieldConfig": {
"defaults": {
+ "color": {
+ "fixedColor": "blue",
+ "mode": "thresholds"
+ },
"decimals": 0,
"links": [ ],
"mappings": [ ],
+ "max": 85,
+ "min": 0,
"thresholds": {
"mode": "absolute",
- "steps": [ ]
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "#EAB839",
+ "value": 70
+ },
+ {
+ "color": "semi-dark-red",
+ "value": 78
+ }
+ ]
},
"unit": "celsius"
}
"links": [ ],
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*\"})",
+ "expr": "max(ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"network\"})",
+ "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"network\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"fans\"})",
+ "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"fans\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "max(ceph_node_proxy_health{hostname=~\"$hostname\",category=\"storage\"})",
+ "expr": "max(ceph_hardware_health{hostname=~\"$hostname\",category=\"storage\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "__auto",
"styles": [ ],
"targets": [
{
- "expr": "ceph_node_proxy_storage_capacity_bytes{hostname=~\"$hostname\"}",
+ "expr": "ceph_hardware_storage_capacity_bytes{hostname=~\"$hostname\"}",
"format": "table",
"instant": true,
"intervalFactor": 1,
"styles": [ ],
"targets": [
{
- "expr": "ceph_node_proxy_firmware_info{hostname=~\"$hostname\"}",
+ "expr": "ceph_hardware_firmware_info{hostname=~\"$hostname\"}",
"format": "table",
"instant": true,
"intervalFactor": 1,
"description": "",
"fieldConfig": {
"defaults": {
+ "color": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ },
"decimals": 0,
"links": [ ],
"mappings": [ ],
"maxPerRow": 6,
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"repeatDirection": "h",
"targets": [
{
- "expr": "ceph_node_proxy_fan_rpm{hostname=~\"$hostname\",fan_name=~\"$fan_speeds\"}",
+ "expr": "ceph_hardware_fan_rpm{hostname=~\"$hostname\",fan_name=~\"$fan_speeds\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{fan_name}}",
"description": "",
"fieldConfig": {
"defaults": {
+ "color": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ },
"decimals": 0,
"links": [ ],
"mappings": [ ],
"links": [ ],
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "ceph_node_proxy_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU1.*\"}",
+ "expr": "ceph_hardware_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU1.*\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{fan_name}}",
"description": "",
"fieldConfig": {
"defaults": {
+ "color": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ },
"decimals": 0,
"links": [ ],
"mappings": [ ],
"links": [ ],
"options": {
"colorMode": "none",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"pluginVersion": "9.1.3",
"targets": [
{
- "expr": "ceph_node_proxy_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU2.*\"}",
+ "expr": "ceph_hardware_fan_rpm{hostname=~\"$hostname\",fan_name=~\"PSU2.*\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{fan_name}}",
"steppedLine": false,
"targets": [
{
- "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"}",
+ "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*CPU_TEMP\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{sensor_name}}",
"steppedLine": false,
"targets": [
{
- "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"}",
+ "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*DIMM.*_TEMP\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{sensor_name}}",
"steppedLine": false,
"targets": [
{
- "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"}",
+ "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\".*MB_TEMP.*\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{sensor_name}}",
"steppedLine": false,
"targets": [
{
- "expr": "ceph_node_proxy_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*_TEMP\"}",
+ "expr": "ceph_hardware_temperature_celsius{hostname=~\"$hostname\", sensor_name=~\"NVME.*_TEMP\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{sensor_name}}",
"steppedLine": false,
"targets": [
{
- "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})",
+ "expr": "avg(ceph_hardware_fan_rpm{hostname=~\"$hostname\", fan_name=~\"PSU.*\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "PSU Fans",
},
"yaxes": [
{
- "format": "none",
+ "format": "locale",
"label": null,
"logBase": 1,
"max": null,
"show": true
},
{
- "format": "none",
+ "format": "locale",
"label": null,
"logBase": 1,
"max": null,
"steppedLine": false,
"targets": [
{
- "expr": "avg(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\", fan_name!~\"PSU.*\"})",
+ "expr": "avg(ceph_hardware_fan_rpm{hostname=~\"$hostname\", fan_name!~\"PSU.*\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "System Fans",
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
- "title": "AVG Cooling Fan Speed",
+ "title": "AVG Cooling Fan RPMs",
"tooltip": {
"shared": true,
"sort": 0,
},
"yaxes": [
{
- "format": "none",
+ "format": "locale",
"label": null,
"logBase": 1,
"max": null,
"show": true
},
{
- "format": "none",
+ "format": "locale",
"label": null,
"logBase": 1,
"max": null,
"multi": false,
"name": "hostname",
"options": [ ],
- "query": "label_values(ceph_node_proxy_health, hostname)",
+ "query": "label_values(ceph_hardware_health, hostname)",
"refresh": 1,
"regex": "",
"sort": 1,
"multi": true,
"name": "fan_speeds",
"options": [ ],
- "query": "label_values(ceph_node_proxy_fan_rpm{hostname=~\"$hostname\"},fan_name)",
+ "query": "label_values(ceph_hardware_fan_rpm{hostname=~\"$hostname\"},fan_name)",
"refresh": 1,
- "regex": "",
+ "regex": "/.*TACH_OUT/",
"sort": 0,
"tagValuesQuery": "",
"tags": [ ],
SMB_METADATA = ('smb_version', 'volume',
'subvolume_group', 'subvolume', 'netbiosname', 'share')
-NODE_PROXY_STORAGE_LABELS = ('hostname', 'device', 'model', 'protocol')
+HW_STORAGE_LABELS = ('hostname', 'device', 'model', 'protocol', 'firmware_version', 'slot', 'serial_number')
-NODE_PROXY_CPU_LABELS = ('hostname', 'cpu', 'manufacturer', 'model')
+HW_CPU_LABELS = ('hostname', 'cpu', 'manufacturer', 'model', 'total_threads')
-NODE_PROXY_MEMORY_LABELS = ('hostname', 'dimm', 'type', 'manufacturer')
+HW_MEMORY_LABELS = ('hostname', 'dimm', 'type')
MIB_TO_BYTES = 1048576
-NODE_PROXY_HEALTH_LABELS = ('hostname', 'component', 'category')
+HW_HEALTH_LABELS = ('hostname', 'component', 'category')
-NODE_PROXY_TEMP_LABELS = ('hostname', 'sensor_name')
+HW_TEMP_LABELS = ('hostname', 'sensor_name')
-NODE_PROXY_FAN_LABELS = ('hostname', 'fan_name')
+HW_FAN_LABELS = ('hostname', 'fan_name')
-NODE_PROXY_FIRMWARE_LABELS = ('hostname', 'component', 'version')
+HW_FIRMWARE_LABELS = ('hostname', 'component', 'version')
+
+HEALTH_STATUS_MAP = {
+ 'OK': 0,
+ 'Warning': 1,
+ 'Degraded': 1,
+ 'Error': 2,
+ 'Critical': 2,
+}
+
+SENSOR_METRICS = {
+ 'fans': {
+ 'metric': 'hardware_fan_rpm',
+ 'description': 'Hardware fan speed in RPM',
+ 'labels': HW_FAN_LABELS,
+ },
+ 'temperatures': {
+ 'metric': 'hardware_temperature_celsius',
+ 'description': 'Hardware temperature sensor reading in Celsius',
+ 'labels': HW_TEMP_LABELS,
+ },
+}
CEPHADM_DAEMON_STATUS = ('service_type', 'daemon_name', 'hostname', 'service_name')
check.description,
)
- metrics['node_proxy_storage_capacity_bytes'] = Metric(
+ metrics['hardware_storage_capacity_bytes'] = Metric(
'gauge',
- 'node_proxy_storage_capacity_bytes',
+ 'hardware_storage_capacity_bytes',
'Storage device capacity in bytes',
- NODE_PROXY_STORAGE_LABELS
+ HW_STORAGE_LABELS
)
- metrics['node_proxy_cpu_cores'] = Metric(
+ metrics['hardware_cpu_cores'] = Metric(
'gauge',
- 'node_proxy_cpu_cores',
+ 'hardware_cpu_cores',
'Total CPU cores',
- NODE_PROXY_CPU_LABELS
+ HW_CPU_LABELS
)
- metrics['node_proxy_memory_capacity_bytes'] = Metric(
+ metrics['hardware_memory_capacity_bytes'] = Metric(
'gauge',
- 'node_proxy_memory_capacity_bytes',
+ 'hardware_memory_capacity_bytes',
'Memory capacity in bytes',
- NODE_PROXY_MEMORY_LABELS
+ HW_MEMORY_LABELS
)
- metrics['node_proxy_health'] = Metric(
+ metrics['hardware_health'] = Metric(
'gauge',
- 'node_proxy_health',
+ 'hardware_health',
'Hardware component health status (0=OK, 1=Warning, 2=Error)',
- NODE_PROXY_HEALTH_LABELS
- )
-
- metrics['node_proxy_temperature_celsius'] = Metric(
- 'gauge',
- 'node_proxy_temperature_celsius',
- 'Hardware temperature sensor reading in Celsius',
- NODE_PROXY_TEMP_LABELS
+ HW_HEALTH_LABELS
)
- metrics['node_proxy_fan_rpm'] = Metric(
- 'gauge',
- 'node_proxy_fan_rpm',
- 'Hardware fan speed in RPM',
- NODE_PROXY_FAN_LABELS
- )
+ for sensor in SENSOR_METRICS.values():
+ metrics[sensor['metric']] = Metric(
+ 'gauge',
+ sensor['metric'],
+ sensor['description'],
+ sensor['labels']
+ )
- metrics['node_proxy_firmware_info'] = Metric(
+ metrics['hardware_firmware_info'] = Metric(
'gauge',
- 'node_proxy_firmware_info',
+ 'hardware_firmware_info',
'Firmware version information (value is always 1, version in label)',
- NODE_PROXY_FIRMWARE_LABELS
+ HW_FIRMWARE_LABELS
)
return metrics
except Exception as e:
self.log.error(f"Failed to get SMB metadata: {str(e)}")
+ def _hw_get_health_value(self, status_val, hostname='', comp_id='', category=''):
+ """Map a Redfish health status to a numeric value (0=OK, 1=Warning, 2=Error)."""
+ if isinstance(status_val, dict):
+ health_str = status_val.get('health', 'Unknown')
+ elif isinstance(status_val, str):
+ health_str = status_val
+ else:
+ health_str = 'Unknown'
+
+ value = HEALTH_STATUS_MAP.get(health_str)
+ if value is None:
+ self.log.warning(
+ f"node-proxy: unexpected health status "
+ f"'{health_str}' for {category}/{comp_id} "
+ f"on {hostname}, skipping"
+ )
+ return value
+
+ def _hw_set_health_metric(self, status_val, hostname, comp_id, category):
+ """Set ceph_hardware_health gauge for a single component."""
+ health_value = self._hw_get_health_value(
+ status_val, hostname, comp_id, category
+ )
+ if health_value is not None:
+ self.metrics['hardware_health'].set(
+ health_value, (hostname, comp_id, category)
+ )
+
+ def _hw_iter_components(self, status, category):
+ """Yield (comp_id, comp_dict) pairs from nested status[category] dicts."""
+ for components in status.get(category, {}).values():
+ for comp_id, comp in components.items():
+ if isinstance(comp, dict):
+ yield comp_id, comp
+
+ def _hw_set_sensor_metric(self, comp, comp_id, hostname, category, metric_key):
+ """Set sensor value (temperature/fan) and health gauges using the sensor name."""
+ name = comp.get('name', comp_id)
+ reading = comp.get('reading')
+ if reading is not None and reading != 'unknown':
+ try:
+ self.metrics[metric_key].set(float(reading), (hostname, name))
+ except (ValueError, TypeError):
+ pass
+ self._hw_set_health_metric(comp.get('status', {}), hostname, name, category)
+
+ def _process_storage(self, status, hostname):
+ """Set storage capacity and health metrics per drive."""
+ for device_id, device in self._hw_iter_components(status, 'storage'):
+ capacity = device.get('capacity_bytes', 0)
+ labels = (
+ hostname,
+ device_id,
+ device.get('model', 'unknown'),
+ device.get('protocol', 'unknown'),
+ device.get('firmware_version', 'unknown'),
+ device.get('slot', 'unknown'),
+ device.get('serial_number', 'unknown')
+ )
+ self.metrics['hardware_storage_capacity_bytes'].set(capacity, labels)
+ self._hw_set_health_metric(device.get('status', {}), hostname, device_id, 'storage')
+
+ def _process_processors(self, status, hostname):
+ """Set CPU cores count and health metrics per processor."""
+ for cpu_id, cpu in self._hw_iter_components(status, 'processors'):
+ cores = cpu.get('total_cores', 0)
+ labels = (
+ hostname,
+ cpu_id,
+ cpu.get('manufacturer', 'unknown'),
+ cpu.get('model', 'unknown'),
+ cpu.get('total_threads', 'unknown')
+ )
+ self.metrics['hardware_cpu_cores'].set(cores, labels)
+ self._hw_set_health_metric(cpu.get('status', {}), hostname, cpu_id, 'processors')
+
+ def _process_memory(self, status, hostname):
+ """Set memory capacity (in bytes) and health metrics per DIMM."""
+ for dimm_id, dimm in self._hw_iter_components(status, 'memory'):
+ capacity_mib = dimm.get('capacity_mi_b', 0)
+ capacity_bytes = capacity_mib * MIB_TO_BYTES
+ labels = (
+ hostname,
+ dimm_id,
+ dimm.get('memory_device_type', 'unknown'),
+ )
+ self.metrics['hardware_memory_capacity_bytes'].set(capacity_bytes, labels)
+ self._hw_set_health_metric(dimm.get('status', {}), hostname, dimm_id, 'memory')
+
+ def _process_power_network(self, status, hostname):
+ """Set health metrics for power and network"""
+ for comp_id, comp in self._hw_iter_components(status, 'power'):
+ name = comp.get('name', comp_id)
+ self._hw_set_health_metric(comp.get('status', {}), hostname, name, 'power')
+ for comp_id, comp in self._hw_iter_components(status, 'network'):
+ self._hw_set_health_metric(comp.get('status', {}), hostname, comp_id, 'network')
+
+ def _process_sensors(self, status, hostname):
+ """Set fan and temperature readings and health metrics"""
+ for category, sensor in SENSOR_METRICS.items():
+ for comp_id, comp in self._hw_iter_components(status, category):
+ self._hw_set_sensor_metric(comp, comp_id, hostname, category, sensor['metric'])
+
+ def _process_firmware(self, hostname, data):
+ """Set firmware info metrics (value=1) with version as label, skip unknown."""
+ fw_data = data.get('firmware', data.get('firmwares', {}))
+ for fw_id, fw_info in fw_data.items():
+ component = fw_info.get('name', fw_id)
+ version = fw_info.get('version', 'unknown')
+ if version and version != 'unknown':
+ labels = (hostname, component, str(version))
+ self.metrics['hardware_firmware_info'].set(1, labels)
+
@profile_method(True)
def get_hardware_metrics(self) -> None:
+ """Fetch node-proxy fullreport and export all hardware metrics."""
if not self.orch_is_available():
return
try:
report = raise_if_exception(self.node_proxy_fullreport())
- firmware_report = raise_if_exception(self.node_proxy_firmware())
except Exception as e:
self.log.debug(f"node-proxy data not available: {e}")
return
- for host, data in report.items():
+ for hostname, data in report.items():
status = data.get('status', {})
-
- for sys_id, devices in status.get('storage', {}).items():
- for device_id, device in devices.items():
- capacity = device.get('capacity_bytes', 0)
- labels = (
- host,
- device_id,
- device.get('model', 'unknown'),
- device.get('protocol', 'unknown')
- )
- self.metrics['node_proxy_storage_capacity_bytes'].set(capacity, labels)
-
- for sys_id, cpus in status.get('processors', {}).items():
- for cpu_id, cpu in cpus.items():
- cores = cpu.get('total_cores', 0)
- labels = (
- host,
- cpu_id,
- cpu.get('manufacturer', 'unknown'),
- cpu.get('model', 'unknown')
- )
- self.metrics['node_proxy_cpu_cores'].set(cores, labels)
-
- for sys_id, dimms in status.get('memory', {}).items():
- for dimm_id, dimm in dimms.items():
- capacity_mib = dimm.get('capacity_mi_b', 0)
- capacity_bytes = capacity_mib * MIB_TO_BYTES
- labels = (
- host,
- dimm_id,
- dimm.get('memory_device_type', 'unknown'),
- dimm.get('manufacturer', 'unknown')
- )
- self.metrics['node_proxy_memory_capacity_bytes'].set(capacity_bytes, labels)
-
- for category in ['storage', 'processors', 'memory', 'power', 'fans', 'network', 'temperatures']:
- for sys_id, components in status.get(category, {}).items():
- for comp_id, comp in components.items():
- if not isinstance(comp, dict):
- continue
- status_val = comp.get('status', {})
- if isinstance(status_val, dict):
- health_str = status_val.get('health', 'Unknown')
- elif isinstance(status_val, str):
- health_str = status_val
- else:
- health_str = 'Unknown'
-
- if health_str == 'OK':
- health_value = 0
- elif health_str in ['Warning', 'Degraded']:
- health_value = 1
- else:
- health_value = 2
-
- labels = (host, comp_id, category)
- self.metrics['node_proxy_health'].set(health_value, labels)
-
- for sys_id, sensors in status.get('temperatures', {}).items():
- for sensor_id, sensor in sensors.items():
- reading = sensor.get('reading')
- sensor_name = sensor.get('name', sensor_id)
- if reading is None or reading == 'unknown':
- continue
- try:
- temp_value = float(reading)
- labels = (host, sensor_name)
- self.metrics['node_proxy_temperature_celsius'].set(temp_value, labels)
- except (ValueError, TypeError):
- continue
-
- for sys_id, fans_data in status.get('fans', {}).items():
- for fan_id, fan in fans_data.items():
- rpm = fan.get('reading')
- fan_name = fan.get('name', fan_id)
- if rpm is None or rpm == 'unknown':
- continue
- try:
- rpm_value = float(rpm)
- labels = (host, fan_name)
- self.metrics['node_proxy_fan_rpm'].set(rpm_value, labels)
- except (ValueError, TypeError):
- continue
-
- for host, fw_data in firmware_report.items():
- for fw_id, fw_info in fw_data.items():
- component = fw_info.get('name', fw_id)
- version = fw_info.get('version', 'unknown')
- if version and version != 'unknown':
- labels = (host, component, str(version))
- self.metrics['node_proxy_firmware_info'].set(1, labels)
+ self._process_firmware(hostname, data)
+ self._process_storage(status, hostname)
+ self._process_processors(status, hostname)
+ self._process_memory(status, hostname)
+ self._process_power_network(status, hostname)
+ self._process_sensors(status, hostname)
@profile_method(True)
def collect(self) -> str:
self.assertEqual(len(cache), 100)
for i in range(100):
self.assertEqual(cache[f'key{i}'], f'value{i}')
+
+
+MOCK_HW_FULLREPORT = {
+ 'at3n2.tuc.stglabs.ibm.com': {
+ 'host': 'at3n2.tuc.stglabs.ibm.com',
+ 'sn': '11S03NK990YM30ZP58E02X',
+ 'status': {
+ 'storage': {'Self': {
+ 'nvme_device0_nsid1': {
+ 'capacity_bytes': 512110190592,
+ 'model': 'Micron_2550_MTFDKBK512TGE',
+ 'protocol': 'NVMe',
+ 'serial_number': '24424BAA3C40',
+ 'firmware_version': 'V6MA001',
+ 'slot': '0',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }},
+ 'processors': {'Self': {
+ 'devtype1_cpu0': {
+ 'total_cores': 48,
+ 'total_threads': 96,
+ 'model': 'unknown',
+ 'manufacturer': 'Advanced Micro Devices, Inc.',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }},
+ 'memory': {'Self': {
+ 'devtype2_dimm0': {
+ 'memory_device_type': 'DDR5',
+ 'capacity_mi_b': 131072,
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }},
+ 'power': {'Self': {
+ '38': {
+ 'name': 'PSU1_PIN',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ '39': {
+ 'name': 'PSU1_POUT',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }},
+ 'network': {'Self': {
+ 'networkinterfaces_devtype7_nic0': {
+ 'name': 'NetworkAdapter_0',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }},
+ 'fans': {'Self': {
+ '0': {
+ 'name': 'FAN1_TACH_IN',
+ 'reading': 23940,
+ 'reading_units': 'RPM',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }},
+ 'temperatures': {'Self': {
+ '0': {
+ 'name': 'C1_DCSCM_TEMP',
+ 'reading': 32,
+ 'reading_units': 'Cel',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ '6': {
+ 'name': 'C1_CPU_TEMP',
+ 'reading': 47,
+ 'reading_units': 'Cel',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }},
+ },
+ 'firmware': {
+ 'bios': {
+ 'name': 'BIOS',
+ 'version': 'unknown',
+ },
+ 'bmc': {
+ 'name': 'BMC',
+ 'version': '0.50.d57cfd',
+ },
+ 'cpld': {
+ 'name': 'CPLD',
+ 'version': 'unknown',
+ },
+ },
+ }
+}
+
+
+class HardwareMetricsTest(TestCase):
+ """Tests for hardware metrics processing from node-proxy fullreport data."""
+
+ def setUp(self):
+ from prometheus.module import (
+ Module, HW_STORAGE_LABELS, HW_CPU_LABELS, HW_MEMORY_LABELS,
+ HW_HEALTH_LABELS, HW_TEMP_LABELS, HW_FAN_LABELS, HW_FIRMWARE_LABELS,
+ )
+ self.module = mock.MagicMock(spec=Module)
+ self.module.log = mock.MagicMock()
+ self.module.metrics = {
+ 'hardware_storage_capacity_bytes': Metric(
+ 'gauge', 'hardware_storage_capacity_bytes', '', HW_STORAGE_LABELS),
+ 'hardware_cpu_cores': Metric(
+ 'gauge', 'hardware_cpu_cores', '', HW_CPU_LABELS),
+ 'hardware_memory_capacity_bytes': Metric(
+ 'gauge', 'hardware_memory_capacity_bytes', '', HW_MEMORY_LABELS),
+ 'hardware_health': Metric(
+ 'gauge', 'hardware_health', '', HW_HEALTH_LABELS),
+ 'hardware_temperature_celsius': Metric(
+ 'gauge', 'hardware_temperature_celsius', '', HW_TEMP_LABELS),
+ 'hardware_fan_rpm': Metric(
+ 'gauge', 'hardware_fan_rpm', '', HW_FAN_LABELS),
+ 'hardware_firmware_info': Metric(
+ 'gauge', 'hardware_firmware_info', '', HW_FIRMWARE_LABELS),
+ }
+ self.hostname = 'at3n2.tuc.stglabs.ibm.com'
+ self.data = MOCK_HW_FULLREPORT[self.hostname]
+ self.status = self.data['status']
+ self.module._hw_get_health_value = Module._hw_get_health_value.__get__(self.module)
+ self.module._hw_set_health_metric = Module._hw_set_health_metric.__get__(self.module)
+ self.module._hw_iter_components = Module._hw_iter_components.__get__(self.module)
+ self.module._hw_set_sensor_metric = Module._hw_set_sensor_metric.__get__(self.module)
+ self.module._process_storage = Module._process_storage.__get__(self.module)
+ self.module._process_processors = Module._process_processors.__get__(self.module)
+ self.module._process_memory = Module._process_memory.__get__(self.module)
+ self.module._process_power_network = Module._process_power_network.__get__(self.module)
+ self.module._process_sensors = Module._process_sensors.__get__(self.module)
+ self.module._process_firmware = Module._process_firmware.__get__(self.module)
+
+ # --- _hw_get_health_value ---
+
+ def test_health_value_ok(self):
+ val = self.module._hw_get_health_value({'health': 'OK'})
+ self.assertEqual(val, 0)
+
+ def test_health_value_warning(self):
+ val = self.module._hw_get_health_value({'health': 'Warning'})
+ self.assertEqual(val, 1)
+
+ def test_health_value_critical(self):
+ val = self.module._hw_get_health_value({'health': 'Critical'})
+ self.assertEqual(val, 2)
+
+ def test_health_value_string_input(self):
+ val = self.module._hw_get_health_value('OK')
+ self.assertEqual(val, 0)
+
+ def test_health_value_unknown_logs_warning(self):
+ val = self.module._hw_get_health_value(
+ {'health': 'Exploded'}, 'host1', 'comp1', 'storage')
+ self.assertIsNone(val)
+ self.module.log.warning.assert_called_once()
+
+ def test_health_value_unknown_skips_metric(self):
+ self.module._hw_set_health_metric(
+ {'health': 'Exploded'}, 'host1', 'comp1', 'storage')
+ self.assertEqual(self.module.metrics['hardware_health'].value, {})
+
+ # --- _process_storage ---
+
+ def test_storage_capacity_and_labels(self):
+ self.module._process_storage(self.status, self.hostname)
+ expected_labels = (
+ self.hostname, 'nvme_device0_nsid1',
+ 'Micron_2550_MTFDKBK512TGE', 'NVMe', 'V6MA001', '0', '24424BAA3C40',
+ )
+ self.assertEqual(
+ self.module.metrics['hardware_storage_capacity_bytes'].value[expected_labels],
+ 512110190592,
+ )
+
+ def test_storage_sets_health(self):
+ self.module._process_storage(self.status, self.hostname)
+ health_labels = (self.hostname, 'nvme_device0_nsid1', 'storage')
+ self.assertEqual(
+ self.module.metrics['hardware_health'].value[health_labels], 0)
+
+ # --- _process_processors ---
+
+ def test_cpu_cores_and_threads_label(self):
+ self.module._process_processors(self.status, self.hostname)
+ expected_labels = (
+ self.hostname, 'devtype1_cpu0',
+ 'Advanced Micro Devices, Inc.', 'unknown', 96,
+ )
+ self.assertEqual(
+ self.module.metrics['hardware_cpu_cores'].value[expected_labels], 48)
+
+ # --- _process_memory ---
+
+ def test_memory_mib_to_bytes_conversion(self):
+ self.module._process_memory(self.status, self.hostname)
+ expected_labels = (self.hostname, 'devtype2_dimm0', 'DDR5')
+ self.assertEqual(
+ self.module.metrics['hardware_memory_capacity_bytes'].value[expected_labels],
+ 131072 * 1048576,
+ )
+
+ # --- _process_power_network ---
+
+ def test_power_uses_name_not_numeric_id(self):
+ """Power components use human-readable name (PSU1_PIN) not dict key (38)."""
+ self.module._process_power_network(self.status, self.hostname)
+ health = self.module.metrics['hardware_health'].value
+ self.assertIn((self.hostname, 'PSU1_PIN', 'power'), health)
+ self.assertIn((self.hostname, 'PSU1_POUT', 'power'), health)
+ self.assertNotIn((self.hostname, '38', 'power'), health)
+ self.assertNotIn((self.hostname, '39', 'power'), health)
+
+ def test_network_uses_comp_id_not_generic_name(self):
+ """Network uses descriptive comp_id, not generic 'NetworkAdapter_0'."""
+ self.module._process_power_network(self.status, self.hostname)
+ health = self.module.metrics['hardware_health'].value
+ self.assertIn(
+ (self.hostname, 'networkinterfaces_devtype7_nic0', 'network'), health)
+ self.assertNotIn(
+ (self.hostname, 'NetworkAdapter_0', 'network'), health)
+
+ # --- _process_sensors ---
+
+ def test_temperature_uses_sensor_name(self):
+ """Temperature reading uses 'C1_DCSCM_TEMP' not numeric key '0'."""
+ self.module._process_sensors(self.status, self.hostname)
+ temp = self.module.metrics['hardware_temperature_celsius'].value
+ self.assertIn((self.hostname, 'C1_DCSCM_TEMP'), temp)
+ self.assertEqual(temp[(self.hostname, 'C1_DCSCM_TEMP')], 32.0)
+ self.assertNotIn((self.hostname, '0'), temp)
+
+ def test_temperature_health_uses_sensor_name(self):
+ """Health metric for temperatures also uses sensor name, not numeric key."""
+ self.module._process_sensors(self.status, self.hostname)
+ health = self.module.metrics['hardware_health'].value
+ self.assertIn((self.hostname, 'C1_DCSCM_TEMP', 'temperatures'), health)
+ self.assertIn((self.hostname, 'C1_CPU_TEMP', 'temperatures'), health)
+ self.assertNotIn((self.hostname, '0', 'temperatures'), health)
+ self.assertNotIn((self.hostname, '6', 'temperatures'), health)
+
+ def test_fan_uses_fan_name(self):
+ """Fan reading uses 'FAN1_TACH_IN' not numeric key '0'."""
+ self.module._process_sensors(self.status, self.hostname)
+ fan = self.module.metrics['hardware_fan_rpm'].value
+ self.assertIn((self.hostname, 'FAN1_TACH_IN'), fan)
+ self.assertEqual(fan[(self.hostname, 'FAN1_TACH_IN')], 23940.0)
+
+ def test_sensor_unknown_reading_skipped(self):
+ """Sensors with 'unknown' reading should not set reading metric but still set health."""
+ status = {'temperatures': {'Self': {
+ '99': {
+ 'name': 'GHOST_TEMP',
+ 'reading': 'unknown',
+ 'status': {'health': 'OK', 'state': 'Enabled'},
+ },
+ }}}
+ self.module._process_sensors(status, self.hostname)
+ self.assertNotIn(
+ (self.hostname, 'GHOST_TEMP'),
+ self.module.metrics['hardware_temperature_celsius'].value)
+ self.assertIn(
+ (self.hostname, 'GHOST_TEMP', 'temperatures'),
+ self.module.metrics['hardware_health'].value)
+
+ # --- _process_firmware ---
+
+ def test_firmware_known_version_exported(self):
+ self.module._process_firmware(self.hostname, self.data)
+ fw = self.module.metrics['hardware_firmware_info'].value
+ self.assertIn((self.hostname, 'BMC', '0.50.d57cfd'), fw)
+ self.assertEqual(fw[(self.hostname, 'BMC', '0.50.d57cfd')], 1)
+
+ def test_firmware_unknown_version_skipped(self):
+ """BIOS and CPLD with version='unknown' should not be exported."""
+ self.module._process_firmware(self.hostname, self.data)
+ fw = self.module.metrics['hardware_firmware_info'].value
+ for labels in fw:
+ self.assertNotEqual(labels[1], 'BIOS')
+ self.assertNotEqual(labels[1], 'CPLD')
+
+ # --- label count consistency ---
+
+ def test_storage_label_count_matches(self):
+ self.module._process_storage(self.status, self.hostname)
+ for labels in self.module.metrics['hardware_storage_capacity_bytes'].value:
+ self.assertEqual(len(labels), 7)
+
+ def test_cpu_label_count_matches(self):
+ self.module._process_processors(self.status, self.hostname)
+ for labels in self.module.metrics['hardware_cpu_cores'].value:
+ self.assertEqual(len(labels), 5)
projects / ceph.git / commitdiff