From: Afreen Misbah Date: Thu, 2 Jul 2026 09:21:19 +0000 (+0530) Subject: mointoring: Add hardware alerts X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=42b3a7dfe820cab0fd5b8af22c88a29c5a1dc546;p=ceph.git mointoring: Add hardware alerts Signed-off-by: Afreen Misbah --- diff --git a/monitoring/ceph-mixin/config.libsonnet b/monitoring/ceph-mixin/config.libsonnet index 180ebfa0903..3bd16bf20a1 100644 --- a/monitoring/ceph-mixin/config.libsonnet +++ b/monitoring/ceph-mixin/config.libsonnet @@ -22,5 +22,11 @@ // Read/Write latency is defined in ms NVMeoFHighClientReadLatency: 10, NVMeoFHighClientWriteLatency: 20, + // + // Hardware temperature thresholds in Celsius + HWTempMotherboardWarning: 60, + HWTempDIMMWarning: 80, + HWTempProcessorWarning: 80, + HWTempNVMeWarning: 70, }, } diff --git a/monitoring/ceph-mixin/prometheus_alerts.libsonnet b/monitoring/ceph-mixin/prometheus_alerts.libsonnet index 6173a9dd781..9e5e4eafd5b 100644 --- a/monitoring/ceph-mixin/prometheus_alerts.libsonnet +++ b/monitoring/ceph-mixin/prometheus_alerts.libsonnet @@ -689,63 +689,163 @@ name: 'hardware', rules: [ { - alert: 'HardwareStorageError', - 'for': '30s', - expr: 'ceph_health_detail{name="HARDWARE_STORAGE"} > 0', + alert: 'PowerSupplyDegraded', + 'for': '1m', + expr: 'ceph_hardware_health{category="power"} == 1', + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'Power supply {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'PSU {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.', + }, + }, + { + alert: 'PowerSupplyFailed', + 'for': '1m', + expr: 'ceph_hardware_health{category="power"} == 2', + labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.5' }, + annotations: { + summary: 'Power supply {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'PSU fault detected on {{ $labels.hostname }}. PSU {{ $labels.component }} has failed.', + }, + }, + { + alert: 'CoolingDegraded', + 'for': '1m', + expr: 'ceph_hardware_health{category="fans"} == 1', + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'Fan {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'Fan module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.', + }, + }, + { + alert: 'CoolingFailed', + 'for': '1m', + expr: 'ceph_hardware_health{category="fans"} == 2', + labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.6' }, + annotations: { + summary: 'Fan {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'A cooling fault has been detected on {{ $labels.hostname }}. A cooling FAN within the enclosure has failed.', + }, + }, + { + alert: 'NVMeDriveDegraded', + 'for': '1m', + expr: 'ceph_hardware_health{category="storage"} == 1', + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'Storage device {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'Storage device {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.', + }, + }, + { + alert: 'NVMeDriveFailed', + 'for': '1m', + expr: 'ceph_hardware_health{category="storage"} == 2', labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.1' }, annotations: { - summary: 'Storage devices error(s) detected%(cluster)s' % $.MultiClusterSummary(), - description: 'Some storage devices are in error. Check `ceph health detail`.', + summary: 'Storage device {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'A drive fault has been detected on {{ $labels.hostname }}. Storage device {{ $labels.component }} has failed.', }, }, { - alert: 'HardwareMemoryError', - 'for': '30s', - expr: 'ceph_health_detail{name="HARDWARE_MEMORY"} > 0', + alert: 'MemoryDegraded', + 'for': '1m', + expr: 'ceph_hardware_health{category="memory"} == 1', + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'Memory module {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'Memory module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.', + }, + }, + { + alert: 'MemoryFailed', + 'for': '1m', + expr: 'ceph_hardware_health{category="memory"} == 2', labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.2' }, annotations: { - summary: 'DIMM error(s) detected%(cluster)s' % $.MultiClusterSummary(), - description: 'DIMM error(s) detected. Check `ceph health detail`.', + summary: 'Memory module {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'A memory fault has been detected on {{ $labels.hostname }}. Memory module {{ $labels.component }} has failed.', }, }, { - alert: 'HardwareProcessorError', - 'for': '30s', - expr: 'ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0', + alert: 'ProcessorDegraded', + 'for': '1m', + expr: 'ceph_hardware_health{category="processors"} == 1', + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'Processor {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'Processor {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.', + }, + }, + { + alert: 'ProcessorFailed', + 'for': '1m', + expr: 'ceph_hardware_health{category="processors"} == 2', labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.3' }, annotations: { - summary: 'Processor error(s) detected%(cluster)s' % $.MultiClusterSummary(), - description: 'Processor error(s) detected. Check `ceph health detail`.', + summary: 'Processor {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'A processor fault has been detected on {{ $labels.hostname }}. Processor {{ $labels.component }} has failed.', }, }, { - alert: 'HardwareNetworkError', - 'for': '30s', - expr: 'ceph_health_detail{name="HARDWARE_NETWORK"} > 0', + alert: 'NetworkDegraded', + 'for': '1m', + expr: 'ceph_hardware_health{category="network"} == 1', + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'Network interface {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'Network interface {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.', + }, + }, + { + alert: 'NetworkFailed', + 'for': '1m', + expr: 'ceph_hardware_health{category="network"} == 2', labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.4' }, annotations: { - summary: 'Network error(s) detected%(cluster)s' % $.MultiClusterSummary(), - description: 'Network error(s) detected. Check `ceph health detail`.', + summary: 'Network interface {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(), + description: 'A network fault has been detected on {{ $labels.hostname }}. Network interface {{ $labels.component }} has failed.', }, }, { - alert: 'HardwarePowerError', - 'for': '30s', - expr: 'ceph_health_detail{name="HARDWARE_POWER"} > 0', - labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.5' }, + alert: 'MotherboardTemperatureHigh', + 'for': '1m', + expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*MB_TEMP.*"} > %d' % $._config.HWTempMotherboardWarning, + labels: { severity: 'warning', type: 'ceph_default' }, annotations: { - summary: 'Power supply error(s) detected%(cluster)s' % $.MultiClusterSummary(), - description: 'Power supply error(s) detected. Check `ceph health detail`.', + summary: 'Motherboard sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempMotherboardWarning, $.MultiClusterSummary()], + description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.', }, }, { - alert: 'HardwareFanError', - 'for': '30s', - expr: 'ceph_health_detail{name="HARDWARE_FANS"} > 0', - labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.6' }, + alert: 'DIMMTemperatureHigh', + 'for': '1m', + expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*DIMM.*_TEMP"} > %d' % $._config.HWTempDIMMWarning, + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'DIMM sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempDIMMWarning, $.MultiClusterSummary()], + description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.', + }, + }, + { + alert: 'ProcessorTemperatureHigh', + 'for': '1m', + expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"} > %d' % $._config.HWTempProcessorWarning, + labels: { severity: 'warning', type: 'ceph_default' }, + annotations: { + summary: 'CPU sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempProcessorWarning, $.MultiClusterSummary()], + description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.', + }, + }, + { + alert: 'NVMeTemperatureHigh', + 'for': '1m', + expr: 'ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"} > %d' % $._config.HWTempNVMeWarning, + labels: { severity: 'warning', type: 'ceph_default' }, annotations: { - summary: 'Fan error(s) detected%(cluster)s' % $.MultiClusterSummary(), - description: 'Fan error(s) detected. Check `ceph health detail`.', + summary: 'NVMe sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempNVMeWarning, $.MultiClusterSummary()], + description: 'An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise.', }, }, ], diff --git a/monitoring/ceph-mixin/prometheus_alerts.yml b/monitoring/ceph-mixin/prometheus_alerts.yml index f2aa4defabd..4d3f6d66a61 100644 --- a/monitoring/ceph-mixin/prometheus_alerts.yml +++ b/monitoring/ceph-mixin/prometheus_alerts.yml @@ -650,65 +650,155 @@ groups: type: "ceph_default" - name: "hardware" rules: - - alert: "HardwareStorageError" + - alert: "PowerSupplyDegraded" annotations: - description: "Some storage devices are in error. Check `ceph health detail`." - summary: "Storage devices error(s) detected on cluster {{ $labels.cluster }}" - expr: "ceph_health_detail{name=\"HARDWARE_STORAGE\"} > 0" - for: "30s" + description: "PSU {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}." + summary: "Power supply {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"power\"} == 1" + for: "1m" + labels: + severity: "warning" + type: "ceph_default" + - alert: "PowerSupplyFailed" + annotations: + description: "PSU fault detected on {{ $labels.hostname }}. PSU {{ $labels.component }} has failed." + summary: "Power supply {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"power\"} == 2" + for: "1m" + labels: + oid: "1.3.6.1.4.1.50495.1.2.1.13.5" + severity: "critical" + type: "ceph_default" + - alert: "CoolingDegraded" + annotations: + description: "Fan module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}." + summary: "Fan {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"fans\"} == 1" + for: "1m" + labels: + severity: "warning" + type: "ceph_default" + - alert: "CoolingFailed" + annotations: + description: "A cooling fault has been detected on {{ $labels.hostname }}. A cooling FAN within the enclosure has failed." + summary: "Fan {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"fans\"} == 2" + for: "1m" + labels: + oid: "1.3.6.1.4.1.50495.1.2.1.13.6" + severity: "critical" + type: "ceph_default" + - alert: "NVMeDriveDegraded" + annotations: + description: "Storage device {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}." + summary: "Storage device {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"storage\"} == 1" + for: "1m" + labels: + severity: "warning" + type: "ceph_default" + - alert: "NVMeDriveFailed" + annotations: + description: "A drive fault has been detected on {{ $labels.hostname }}. Storage device {{ $labels.component }} has failed." + summary: "Storage device {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"storage\"} == 2" + for: "1m" labels: oid: "1.3.6.1.4.1.50495.1.2.1.13.1" severity: "critical" type: "ceph_default" - - alert: "HardwareMemoryError" + - alert: "MemoryDegraded" annotations: - description: "DIMM error(s) detected. Check `ceph health detail`." - summary: "DIMM error(s) detected on cluster {{ $labels.cluster }}" - expr: "ceph_health_detail{name=\"HARDWARE_MEMORY\"} > 0" - for: "30s" + description: "Memory module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}." + summary: "Memory module {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"memory\"} == 1" + for: "1m" + labels: + severity: "warning" + type: "ceph_default" + - alert: "MemoryFailed" + annotations: + description: "A memory fault has been detected on {{ $labels.hostname }}. Memory module {{ $labels.component }} has failed." + summary: "Memory module {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"memory\"} == 2" + for: "1m" labels: oid: "1.3.6.1.4.1.50495.1.2.1.13.2" severity: "critical" type: "ceph_default" - - alert: "HardwareProcessorError" + - alert: "ProcessorDegraded" annotations: - description: "Processor error(s) detected. Check `ceph health detail`." - summary: "Processor error(s) detected on cluster {{ $labels.cluster }}" - expr: "ceph_health_detail{name=\"HARDWARE_PROCESSOR\"} > 0" - for: "30s" + description: "Processor {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}." + summary: "Processor {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"processors\"} == 1" + for: "1m" + labels: + severity: "warning" + type: "ceph_default" + - alert: "ProcessorFailed" + annotations: + description: "A processor fault has been detected on {{ $labels.hostname }}. Processor {{ $labels.component }} has failed." + summary: "Processor {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"processors\"} == 2" + for: "1m" labels: oid: "1.3.6.1.4.1.50495.1.2.1.13.3" severity: "critical" type: "ceph_default" - - alert: "HardwareNetworkError" + - alert: "NetworkDegraded" annotations: - description: "Network error(s) detected. Check `ceph health detail`." - summary: "Network error(s) detected on cluster {{ $labels.cluster }}" - expr: "ceph_health_detail{name=\"HARDWARE_NETWORK\"} > 0" - for: "30s" + description: "Network interface {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}." + summary: "Network interface {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"network\"} == 1" + for: "1m" + labels: + severity: "warning" + type: "ceph_default" + - alert: "NetworkFailed" + annotations: + description: "A network fault has been detected on {{ $labels.hostname }}. Network interface {{ $labels.component }} has failed." + summary: "Network interface {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_health{category=\"network\"} == 2" + for: "1m" labels: oid: "1.3.6.1.4.1.50495.1.2.1.13.4" severity: "critical" type: "ceph_default" - - alert: "HardwarePowerError" + - alert: "MotherboardTemperatureHigh" annotations: - description: "Power supply error(s) detected. Check `ceph health detail`." - summary: "Power supply error(s) detected on cluster {{ $labels.cluster }}" - expr: "ceph_health_detail{name=\"HARDWARE_POWER\"} > 0" - for: "30s" + description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall." + summary: "Motherboard sensor {{ $labels.sensor_name }} above 60°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*MB_TEMP.*\"} > 60" + for: "1m" labels: - oid: "1.3.6.1.4.1.50495.1.2.1.13.5" - severity: "critical" + severity: "warning" type: "ceph_default" - - alert: "HardwareFanError" + - alert: "DIMMTemperatureHigh" annotations: - description: "Fan error(s) detected. Check `ceph health detail`." - summary: "Fan error(s) detected on cluster {{ $labels.cluster }}" - expr: "ceph_health_detail{name=\"HARDWARE_FANS\"} > 0" - for: "30s" + description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall." + summary: "DIMM sensor {{ $labels.sensor_name }} above 80°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*DIMM.*_TEMP\"} > 80" + for: "1m" labels: - oid: "1.3.6.1.4.1.50495.1.2.1.13.6" - severity: "critical" + severity: "warning" + type: "ceph_default" + - alert: "ProcessorTemperatureHigh" + annotations: + description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall." + summary: "CPU sensor {{ $labels.sensor_name }} above 80°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*CPU_TEMP\"} > 80" + for: "1m" + labels: + severity: "warning" + type: "ceph_default" + - alert: "NVMeTemperatureHigh" + annotations: + description: "An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise." + summary: "NVMe sensor {{ $labels.sensor_name }} above 70°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}" + expr: "ceph_hardware_temperature_celsius{sensor_name=~\"NVME.*_TEMP\"} > 70" + for: "1m" + labels: + severity: "warning" type: "ceph_default" - name: "PrometheusServer" rules: diff --git a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml index 6a4c2cd2076..d40963ed1e6 100644 --- a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml +++ b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml @@ -2087,156 +2087,440 @@ tests: summary: "The replication network usage on cluster mycluster has been increased over 80% in the last 30 minutes. Review the number of images being replicated. This alert will be cleaned automatically after 30 minutes" description: "Detected a heavy increase in bandwidth for rbd replications (over 80%) in the last 30 min. This might not be a problem, but it is good to review the number of images being replicated simultaneously" - - interval: 30s + # Power supply degraded + - interval: 1m input_series: - - series: 'ceph_health_detail{name="HARDWARE_STORAGE", cluster="mycluster"}' - values: '1+0x40' + - series: 'ceph_hardware_health{category="power",component="PSU0",hostname="node1",cluster="mycluster"}' + values: '1 1 1 1 1 1 1 1 1' promql_expr_test: - - expr: ceph_health_detail{name="HARDWARE_STORAGE"} > 0 + - expr: ceph_hardware_health{category="power"} == 1 eval_time: 2m exp_samples: - - labels: '{__name__="ceph_health_detail", name="HARDWARE_STORAGE", cluster="mycluster"}' + - labels: 'ceph_hardware_health{category="power",component="PSU0",hostname="node1",cluster="mycluster"}' value: 1 alert_rule_test: - eval_time: 1m - alertname: HardwareStorageError + alertname: PowerSupplyDegraded - eval_time: 5m - alertname: HardwareStorageError + alertname: PowerSupplyDegraded exp_alerts: - exp_labels: - name: HARDWARE_STORAGE + category: power + component: PSU0 + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "Power supply PSU0 degraded on node1 on cluster mycluster" + description: "PSU PSU0 is in a degraded state on node1." + # Power supply failed + - interval: 1m + input_series: + - series: 'ceph_hardware_health{category="power",component="PSU1",hostname="node2",cluster="mycluster"}' + values: '2 2 2 2 2 2 2 2 2' + promql_expr_test: + - expr: ceph_hardware_health{category="power"} == 2 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_health{category="power",component="PSU1",hostname="node2",cluster="mycluster"}' + value: 2 + alert_rule_test: + - eval_time: 1m + alertname: PowerSupplyFailed + - eval_time: 5m + alertname: PowerSupplyFailed + exp_alerts: + - exp_labels: + category: power + component: PSU1 + hostname: node2 severity: critical cluster: mycluster type: ceph_default - oid: 1.3.6.1.4.1.50495.1.2.1.13.1 + oid: 1.3.6.1.4.1.50495.1.2.1.13.5 exp_annotations: - summary: Storage devices error(s) detected on cluster mycluster - description: "Some storage devices are in error. Check `ceph health detail`." - - interval: 30s + summary: "Power supply PSU1 failed on node2 on cluster mycluster" + description: "PSU fault detected on node2. PSU PSU1 has failed." + # Cooling degraded + - interval: 1m input_series: - - series: 'ceph_health_detail{name="HARDWARE_MEMORY", cluster="mycluster"}' - values: '1+0x40' + - series: 'ceph_hardware_health{category="fans",component="FAN0",hostname="node1",cluster="mycluster"}' + values: '1 1 1 1 1 1 1 1 1' promql_expr_test: - - expr: ceph_health_detail{name="HARDWARE_MEMORY"} > 0 + - expr: ceph_hardware_health{category="fans"} == 1 eval_time: 2m exp_samples: - - labels: '{__name__="ceph_health_detail", name="HARDWARE_MEMORY", cluster="mycluster"}' + - labels: 'ceph_hardware_health{category="fans",component="FAN0",hostname="node1",cluster="mycluster"}' value: 1 alert_rule_test: - eval_time: 1m - alertname: HardwareMemoryError + alertname: CoolingDegraded - eval_time: 5m - alertname: HardwareMemoryError + alertname: CoolingDegraded exp_alerts: - exp_labels: - name: HARDWARE_MEMORY + category: fans + component: FAN0 + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "Fan FAN0 degraded on node1 on cluster mycluster" + description: "Fan module FAN0 is in a degraded state on node1." + # Cooling failed + - interval: 1m + input_series: + - series: 'ceph_hardware_health{category="fans",component="FAN1",hostname="node2",cluster="mycluster"}' + values: '2 2 2 2 2 2 2 2 2' + promql_expr_test: + - expr: ceph_hardware_health{category="fans"} == 2 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_health{category="fans",component="FAN1",hostname="node2",cluster="mycluster"}' + value: 2 + alert_rule_test: + - eval_time: 1m + alertname: CoolingFailed + - eval_time: 5m + alertname: CoolingFailed + exp_alerts: + - exp_labels: + category: fans + component: FAN1 + hostname: node2 severity: critical cluster: mycluster type: ceph_default - oid: 1.3.6.1.4.1.50495.1.2.1.13.2 + oid: 1.3.6.1.4.1.50495.1.2.1.13.6 exp_annotations: - summary: DIMM error(s) detected on cluster mycluster - description: "DIMM error(s) detected. Check `ceph health detail`." - - interval: 30s + summary: "Fan FAN1 failed on node2 on cluster mycluster" + description: "A cooling fault has been detected on node2. A cooling FAN within the enclosure has failed." + # NVMe drive degraded + - interval: 1m input_series: - - series: 'ceph_health_detail{name="HARDWARE_PROCESSOR", cluster="mycluster"}' - values: '1+0x40' + - series: 'ceph_hardware_health{category="storage",component="nvme0n1",hostname="node1",cluster="mycluster"}' + values: '1 1 1 1 1 1 1 1 1' promql_expr_test: - - expr: ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0 + - expr: ceph_hardware_health{category="storage"} == 1 eval_time: 2m exp_samples: - - labels: '{__name__="ceph_health_detail", name="HARDWARE_PROCESSOR", cluster="mycluster"}' + - labels: 'ceph_hardware_health{category="storage",component="nvme0n1",hostname="node1",cluster="mycluster"}' value: 1 alert_rule_test: - eval_time: 1m - alertname: HardwareProcessorError + alertname: NVMeDriveDegraded - eval_time: 5m - alertname: HardwareProcessorError + alertname: NVMeDriveDegraded exp_alerts: - exp_labels: - name: HARDWARE_PROCESSOR + category: storage + component: nvme0n1 + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "Storage device nvme0n1 degraded on node1 on cluster mycluster" + description: "Storage device nvme0n1 is in a degraded state on node1." + # NVMe drive failed + - interval: 1m + input_series: + - series: 'ceph_hardware_health{category="storage",component="nvme1n1",hostname="node2",cluster="mycluster"}' + values: '2 2 2 2 2 2 2 2 2' + promql_expr_test: + - expr: ceph_hardware_health{category="storage"} == 2 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_health{category="storage",component="nvme1n1",hostname="node2",cluster="mycluster"}' + value: 2 + alert_rule_test: + - eval_time: 1m + alertname: NVMeDriveFailed + - eval_time: 5m + alertname: NVMeDriveFailed + exp_alerts: + - exp_labels: + category: storage + component: nvme1n1 + hostname: node2 severity: critical cluster: mycluster type: ceph_default - oid: 1.3.6.1.4.1.50495.1.2.1.13.3 + oid: 1.3.6.1.4.1.50495.1.2.1.13.1 exp_annotations: - summary: Processor error(s) detected on cluster mycluster - description: "Processor error(s) detected. Check `ceph health detail`." - - interval: 30s + summary: "Storage device nvme1n1 failed on node2 on cluster mycluster" + description: "A drive fault has been detected on node2. Storage device nvme1n1 has failed." + # Memory degraded + - interval: 1m input_series: - - series: 'ceph_health_detail{name="HARDWARE_NETWORK", cluster="mycluster"}' - values: '1+0x40' + - series: 'ceph_hardware_health{category="memory",component="DIMM_A0",hostname="node1",cluster="mycluster"}' + values: '1 1 1 1 1 1 1 1 1' promql_expr_test: - - expr: ceph_health_detail{name="HARDWARE_NETWORK"} > 0 + - expr: ceph_hardware_health{category="memory"} == 1 eval_time: 2m exp_samples: - - labels: '{__name__="ceph_health_detail", name="HARDWARE_NETWORK", cluster="mycluster"}' + - labels: 'ceph_hardware_health{category="memory",component="DIMM_A0",hostname="node1",cluster="mycluster"}' value: 1 alert_rule_test: - eval_time: 1m - alertname: HardwareNetworkError + alertname: MemoryDegraded + - eval_time: 5m + alertname: MemoryDegraded + exp_alerts: + - exp_labels: + category: memory + component: DIMM_A0 + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "Memory module DIMM_A0 degraded on node1 on cluster mycluster" + description: "Memory module DIMM_A0 is in a degraded state on node1." + # Memory failed + - interval: 1m + input_series: + - series: 'ceph_hardware_health{category="memory",component="DIMM_B0",hostname="node2",cluster="mycluster"}' + values: '2 2 2 2 2 2 2 2 2' + promql_expr_test: + - expr: ceph_hardware_health{category="memory"} == 2 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_health{category="memory",component="DIMM_B0",hostname="node2",cluster="mycluster"}' + value: 2 + alert_rule_test: + - eval_time: 1m + alertname: MemoryFailed - eval_time: 5m - alertname: HardwareNetworkError + alertname: MemoryFailed exp_alerts: - exp_labels: - name: HARDWARE_NETWORK + category: memory + component: DIMM_B0 + hostname: node2 severity: critical cluster: mycluster type: ceph_default - oid: 1.3.6.1.4.1.50495.1.2.1.13.4 + oid: 1.3.6.1.4.1.50495.1.2.1.13.2 exp_annotations: - summary: Network error(s) detected on cluster mycluster - description: "Network error(s) detected. Check `ceph health detail`." - - interval: 30s + summary: "Memory module DIMM_B0 failed on node2 on cluster mycluster" + description: "A memory fault has been detected on node2. Memory module DIMM_B0 has failed." + # Processor degraded + - interval: 1m input_series: - - series: 'ceph_health_detail{name="HARDWARE_POWER", cluster="mycluster"}' - values: '1+0x40' + - series: 'ceph_hardware_health{category="processors",component="CPU0",hostname="node1",cluster="mycluster"}' + values: '1 1 1 1 1 1 1 1 1' promql_expr_test: - - expr: ceph_health_detail{name="HARDWARE_POWER"} > 0 + - expr: ceph_hardware_health{category="processors"} == 1 eval_time: 2m exp_samples: - - labels: '{__name__="ceph_health_detail", name="HARDWARE_POWER", cluster="mycluster"}' + - labels: 'ceph_hardware_health{category="processors",component="CPU0",hostname="node1",cluster="mycluster"}' value: 1 alert_rule_test: - eval_time: 1m - alertname: HardwarePowerError + alertname: ProcessorDegraded + - eval_time: 5m + alertname: ProcessorDegraded + exp_alerts: + - exp_labels: + category: processors + component: CPU0 + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "Processor CPU0 degraded on node1 on cluster mycluster" + description: "Processor CPU0 is in a degraded state on node1." + # Processor failed + - interval: 1m + input_series: + - series: 'ceph_hardware_health{category="processors",component="CPU1",hostname="node2",cluster="mycluster"}' + values: '2 2 2 2 2 2 2 2 2' + promql_expr_test: + - expr: ceph_hardware_health{category="processors"} == 2 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_health{category="processors",component="CPU1",hostname="node2",cluster="mycluster"}' + value: 2 + alert_rule_test: + - eval_time: 1m + alertname: ProcessorFailed - eval_time: 5m - alertname: HardwarePowerError + alertname: ProcessorFailed exp_alerts: - exp_labels: - name: HARDWARE_POWER + category: processors + component: CPU1 + hostname: node2 severity: critical cluster: mycluster type: ceph_default - oid: 1.3.6.1.4.1.50495.1.2.1.13.5 + oid: 1.3.6.1.4.1.50495.1.2.1.13.3 exp_annotations: - summary: Power supply error(s) detected on cluster mycluster - description: "Power supply error(s) detected. Check `ceph health detail`." - - interval: 30s + summary: "Processor CPU1 failed on node2 on cluster mycluster" + description: "A processor fault has been detected on node2. Processor CPU1 has failed." + # Network degraded + - interval: 1m input_series: - - series: 'ceph_health_detail{name="HARDWARE_FANS", cluster="mycluster"}' - values: '1+0x40' + - series: 'ceph_hardware_health{category="network",component="eth0",hostname="node1",cluster="mycluster"}' + values: '1 1 1 1 1 1 1 1 1' promql_expr_test: - - expr: ceph_health_detail{name="HARDWARE_FANS"} > 0 + - expr: ceph_hardware_health{category="network"} == 1 eval_time: 2m exp_samples: - - labels: '{__name__="ceph_health_detail", name="HARDWARE_FANS", cluster="mycluster"}' + - labels: 'ceph_hardware_health{category="network",component="eth0",hostname="node1",cluster="mycluster"}' value: 1 alert_rule_test: - eval_time: 1m - alertname: HardwareFanError + alertname: NetworkDegraded + - eval_time: 5m + alertname: NetworkDegraded + exp_alerts: + - exp_labels: + category: network + component: eth0 + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "Network interface eth0 degraded on node1 on cluster mycluster" + description: "Network interface eth0 is in a degraded state on node1." + # Network failed + - interval: 1m + input_series: + - series: 'ceph_hardware_health{category="network",component="eth1",hostname="node2",cluster="mycluster"}' + values: '2 2 2 2 2 2 2 2 2' + promql_expr_test: + - expr: ceph_hardware_health{category="network"} == 2 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_health{category="network",component="eth1",hostname="node2",cluster="mycluster"}' + value: 2 + alert_rule_test: + - eval_time: 1m + alertname: NetworkFailed - eval_time: 5m - alertname: HardwareFanError + alertname: NetworkFailed exp_alerts: - exp_labels: - name: HARDWARE_FANS + category: network + component: eth1 + hostname: node2 severity: critical cluster: mycluster type: ceph_default - oid: 1.3.6.1.4.1.50495.1.2.1.13.6 + oid: 1.3.6.1.4.1.50495.1.2.1.13.4 + exp_annotations: + summary: "Network interface eth1 failed on node2 on cluster mycluster" + description: "A network fault has been detected on node2. Network interface eth1 has failed." + # Motherboard temperature high + - interval: 1m + input_series: + - series: 'ceph_hardware_temperature_celsius{sensor_name="MB_TEMP1",hostname="node1",cluster="mycluster"}' + values: '65 65 65 65 65 65 65 65 65' + promql_expr_test: + - expr: ceph_hardware_temperature_celsius{sensor_name=~".*MB_TEMP.*"} > 60 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_temperature_celsius{sensor_name="MB_TEMP1",hostname="node1",cluster="mycluster"}' + value: 65 + alert_rule_test: + - eval_time: 1m + alertname: MotherboardTemperatureHigh + - eval_time: 5m + alertname: MotherboardTemperatureHigh + exp_alerts: + - exp_labels: + sensor_name: MB_TEMP1 + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "Motherboard sensor MB_TEMP1 above 60°C on node1 on cluster mycluster" + description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall." + # DIMM temperature high + - interval: 1m + input_series: + - series: 'ceph_hardware_temperature_celsius{sensor_name="DIMM_A0_TEMP",hostname="node1",cluster="mycluster"}' + values: '85 85 85 85 85 85 85 85 85' + promql_expr_test: + - expr: ceph_hardware_temperature_celsius{sensor_name=~".*DIMM.*_TEMP"} > 80 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_temperature_celsius{sensor_name="DIMM_A0_TEMP",hostname="node1",cluster="mycluster"}' + value: 85 + alert_rule_test: + - eval_time: 1m + alertname: DIMMTemperatureHigh + - eval_time: 5m + alertname: DIMMTemperatureHigh + exp_alerts: + - exp_labels: + sensor_name: DIMM_A0_TEMP + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "DIMM sensor DIMM_A0_TEMP above 80°C on node1 on cluster mycluster" + description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall." + # Processor temperature high + - interval: 1m + input_series: + - series: 'ceph_hardware_temperature_celsius{sensor_name="CPU_TEMP",hostname="node1",cluster="mycluster"}' + values: '85 85 85 85 85 85 85 85 85' + promql_expr_test: + - expr: ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"} > 80 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_temperature_celsius{sensor_name="CPU_TEMP",hostname="node1",cluster="mycluster"}' + value: 85 + alert_rule_test: + - eval_time: 1m + alertname: ProcessorTemperatureHigh + - eval_time: 5m + alertname: ProcessorTemperatureHigh + exp_alerts: + - exp_labels: + sensor_name: CPU_TEMP + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default + exp_annotations: + summary: "CPU sensor CPU_TEMP above 80°C on node1 on cluster mycluster" + description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall." + # NVMe temperature high + - interval: 1m + input_series: + - series: 'ceph_hardware_temperature_celsius{sensor_name="NVME0_TEMP",hostname="node1",cluster="mycluster"}' + values: '75 75 75 75 75 75 75 75 75' + promql_expr_test: + - expr: ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"} > 70 + eval_time: 2m + exp_samples: + - labels: 'ceph_hardware_temperature_celsius{sensor_name="NVME0_TEMP",hostname="node1",cluster="mycluster"}' + value: 75 + alert_rule_test: + - eval_time: 1m + alertname: NVMeTemperatureHigh + - eval_time: 5m + alertname: NVMeTemperatureHigh + exp_alerts: + - exp_labels: + sensor_name: NVME0_TEMP + hostname: node1 + severity: warning + cluster: mycluster + type: ceph_default exp_annotations: - summary: Fan error(s) detected on cluster mycluster - description: "Fan error(s) detected. Check `ceph health detail`." + summary: "NVMe sensor NVME0_TEMP above 70°C on node1 on cluster mycluster" + description: "An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise." # nvmeof Tests # NVMeoFSubsystemNamespaceLimit