// Read/Write latency is defined in ms
NVMeoFHighClientReadLatency: 10,
NVMeoFHighClientWriteLatency: 20,
+ //
+ // Hardware temperature thresholds in Celsius
+ HWTempMotherboardWarning: 60,
+ HWTempDIMMWarning: 80,
+ HWTempProcessorWarning: 80,
+ HWTempNVMeWarning: 70,
},
}
name: 'hardware',
rules: [
{
- alert: 'HardwareStorageError',
- 'for': '30s',
- expr: 'ceph_health_detail{name="HARDWARE_STORAGE"} > 0',
+ alert: 'PowerSupplyDegraded',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="power"} == 1',
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'Power supply {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'PSU {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
+ },
+ },
+ {
+ alert: 'PowerSupplyFailed',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="power"} == 2',
+ labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.5' },
+ annotations: {
+ summary: 'Power supply {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'PSU fault detected on {{ $labels.hostname }}. PSU {{ $labels.component }} has failed.',
+ },
+ },
+ {
+ alert: 'CoolingDegraded',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="fans"} == 1',
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'Fan {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'Fan module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
+ },
+ },
+ {
+ alert: 'CoolingFailed',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="fans"} == 2',
+ labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.6' },
+ annotations: {
+ summary: 'Fan {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'A cooling fault has been detected on {{ $labels.hostname }}. A cooling FAN within the enclosure has failed.',
+ },
+ },
+ {
+ alert: 'NVMeDriveDegraded',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="storage"} == 1',
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'Storage device {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'Storage device {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
+ },
+ },
+ {
+ alert: 'NVMeDriveFailed',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="storage"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.1' },
annotations: {
- summary: 'Storage devices error(s) detected%(cluster)s' % $.MultiClusterSummary(),
- description: 'Some storage devices are in error. Check `ceph health detail`.',
+ summary: 'Storage device {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'A drive fault has been detected on {{ $labels.hostname }}. Storage device {{ $labels.component }} has failed.',
},
},
{
- alert: 'HardwareMemoryError',
- 'for': '30s',
- expr: 'ceph_health_detail{name="HARDWARE_MEMORY"} > 0',
+ alert: 'MemoryDegraded',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="memory"} == 1',
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'Memory module {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'Memory module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
+ },
+ },
+ {
+ alert: 'MemoryFailed',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="memory"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.2' },
annotations: {
- summary: 'DIMM error(s) detected%(cluster)s' % $.MultiClusterSummary(),
- description: 'DIMM error(s) detected. Check `ceph health detail`.',
+ summary: 'Memory module {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'A memory fault has been detected on {{ $labels.hostname }}. Memory module {{ $labels.component }} has failed.',
},
},
{
- alert: 'HardwareProcessorError',
- 'for': '30s',
- expr: 'ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0',
+ alert: 'ProcessorDegraded',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="processors"} == 1',
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'Processor {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'Processor {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
+ },
+ },
+ {
+ alert: 'ProcessorFailed',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="processors"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.3' },
annotations: {
- summary: 'Processor error(s) detected%(cluster)s' % $.MultiClusterSummary(),
- description: 'Processor error(s) detected. Check `ceph health detail`.',
+ summary: 'Processor {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'A processor fault has been detected on {{ $labels.hostname }}. Processor {{ $labels.component }} has failed.',
},
},
{
- alert: 'HardwareNetworkError',
- 'for': '30s',
- expr: 'ceph_health_detail{name="HARDWARE_NETWORK"} > 0',
+ alert: 'NetworkDegraded',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="network"} == 1',
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'Network interface {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'Network interface {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
+ },
+ },
+ {
+ alert: 'NetworkFailed',
+ 'for': '1m',
+ expr: 'ceph_hardware_health{category="network"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.4' },
annotations: {
- summary: 'Network error(s) detected%(cluster)s' % $.MultiClusterSummary(),
- description: 'Network error(s) detected. Check `ceph health detail`.',
+ summary: 'Network interface {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
+ description: 'A network fault has been detected on {{ $labels.hostname }}. Network interface {{ $labels.component }} has failed.',
},
},
{
- alert: 'HardwarePowerError',
- 'for': '30s',
- expr: 'ceph_health_detail{name="HARDWARE_POWER"} > 0',
- labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.5' },
+ alert: 'MotherboardTemperatureHigh',
+ 'for': '1m',
+ expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*MB_TEMP.*"} > %d' % $._config.HWTempMotherboardWarning,
+ labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
- summary: 'Power supply error(s) detected%(cluster)s' % $.MultiClusterSummary(),
- description: 'Power supply error(s) detected. Check `ceph health detail`.',
+ summary: 'Motherboard sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempMotherboardWarning, $.MultiClusterSummary()],
+ description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.',
},
},
{
- alert: 'HardwareFanError',
- 'for': '30s',
- expr: 'ceph_health_detail{name="HARDWARE_FANS"} > 0',
- labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.6' },
+ alert: 'DIMMTemperatureHigh',
+ 'for': '1m',
+ expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*DIMM.*_TEMP"} > %d' % $._config.HWTempDIMMWarning,
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'DIMM sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempDIMMWarning, $.MultiClusterSummary()],
+ description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.',
+ },
+ },
+ {
+ alert: 'ProcessorTemperatureHigh',
+ 'for': '1m',
+ expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"} > %d' % $._config.HWTempProcessorWarning,
+ labels: { severity: 'warning', type: 'ceph_default' },
+ annotations: {
+ summary: 'CPU sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempProcessorWarning, $.MultiClusterSummary()],
+ description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.',
+ },
+ },
+ {
+ alert: 'NVMeTemperatureHigh',
+ 'for': '1m',
+ expr: 'ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"} > %d' % $._config.HWTempNVMeWarning,
+ labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
- summary: 'Fan error(s) detected%(cluster)s' % $.MultiClusterSummary(),
- description: 'Fan error(s) detected. Check `ceph health detail`.',
+ summary: 'NVMe sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempNVMeWarning, $.MultiClusterSummary()],
+ description: 'An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise.',
},
},
],
type: "ceph_default"
- name: "hardware"
rules:
- - alert: "HardwareStorageError"
+ - alert: "PowerSupplyDegraded"
annotations:
- description: "Some storage devices are in error. Check `ceph health detail`."
- summary: "Storage devices error(s) detected on cluster {{ $labels.cluster }}"
- expr: "ceph_health_detail{name=\"HARDWARE_STORAGE\"} > 0"
- for: "30s"
+ description: "PSU {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
+ summary: "Power supply {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"power\"} == 1"
+ for: "1m"
+ labels:
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "PowerSupplyFailed"
+ annotations:
+ description: "PSU fault detected on {{ $labels.hostname }}. PSU {{ $labels.component }} has failed."
+ summary: "Power supply {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"power\"} == 2"
+ for: "1m"
+ labels:
+ oid: "1.3.6.1.4.1.50495.1.2.1.13.5"
+ severity: "critical"
+ type: "ceph_default"
+ - alert: "CoolingDegraded"
+ annotations:
+ description: "Fan module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
+ summary: "Fan {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"fans\"} == 1"
+ for: "1m"
+ labels:
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "CoolingFailed"
+ annotations:
+ description: "A cooling fault has been detected on {{ $labels.hostname }}. A cooling FAN within the enclosure has failed."
+ summary: "Fan {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"fans\"} == 2"
+ for: "1m"
+ labels:
+ oid: "1.3.6.1.4.1.50495.1.2.1.13.6"
+ severity: "critical"
+ type: "ceph_default"
+ - alert: "NVMeDriveDegraded"
+ annotations:
+ description: "Storage device {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
+ summary: "Storage device {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"storage\"} == 1"
+ for: "1m"
+ labels:
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "NVMeDriveFailed"
+ annotations:
+ description: "A drive fault has been detected on {{ $labels.hostname }}. Storage device {{ $labels.component }} has failed."
+ summary: "Storage device {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"storage\"} == 2"
+ for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.1"
severity: "critical"
type: "ceph_default"
- - alert: "HardwareMemoryError"
+ - alert: "MemoryDegraded"
annotations:
- description: "DIMM error(s) detected. Check `ceph health detail`."
- summary: "DIMM error(s) detected on cluster {{ $labels.cluster }}"
- expr: "ceph_health_detail{name=\"HARDWARE_MEMORY\"} > 0"
- for: "30s"
+ description: "Memory module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
+ summary: "Memory module {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"memory\"} == 1"
+ for: "1m"
+ labels:
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "MemoryFailed"
+ annotations:
+ description: "A memory fault has been detected on {{ $labels.hostname }}. Memory module {{ $labels.component }} has failed."
+ summary: "Memory module {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"memory\"} == 2"
+ for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.2"
severity: "critical"
type: "ceph_default"
- - alert: "HardwareProcessorError"
+ - alert: "ProcessorDegraded"
annotations:
- description: "Processor error(s) detected. Check `ceph health detail`."
- summary: "Processor error(s) detected on cluster {{ $labels.cluster }}"
- expr: "ceph_health_detail{name=\"HARDWARE_PROCESSOR\"} > 0"
- for: "30s"
+ description: "Processor {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
+ summary: "Processor {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"processors\"} == 1"
+ for: "1m"
+ labels:
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "ProcessorFailed"
+ annotations:
+ description: "A processor fault has been detected on {{ $labels.hostname }}. Processor {{ $labels.component }} has failed."
+ summary: "Processor {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"processors\"} == 2"
+ for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.3"
severity: "critical"
type: "ceph_default"
- - alert: "HardwareNetworkError"
+ - alert: "NetworkDegraded"
annotations:
- description: "Network error(s) detected. Check `ceph health detail`."
- summary: "Network error(s) detected on cluster {{ $labels.cluster }}"
- expr: "ceph_health_detail{name=\"HARDWARE_NETWORK\"} > 0"
- for: "30s"
+ description: "Network interface {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
+ summary: "Network interface {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"network\"} == 1"
+ for: "1m"
+ labels:
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "NetworkFailed"
+ annotations:
+ description: "A network fault has been detected on {{ $labels.hostname }}. Network interface {{ $labels.component }} has failed."
+ summary: "Network interface {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_health{category=\"network\"} == 2"
+ for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.4"
severity: "critical"
type: "ceph_default"
- - alert: "HardwarePowerError"
+ - alert: "MotherboardTemperatureHigh"
annotations:
- description: "Power supply error(s) detected. Check `ceph health detail`."
- summary: "Power supply error(s) detected on cluster {{ $labels.cluster }}"
- expr: "ceph_health_detail{name=\"HARDWARE_POWER\"} > 0"
- for: "30s"
+ description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
+ summary: "Motherboard sensor {{ $labels.sensor_name }} above 60°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*MB_TEMP.*\"} > 60"
+ for: "1m"
labels:
- oid: "1.3.6.1.4.1.50495.1.2.1.13.5"
- severity: "critical"
+ severity: "warning"
type: "ceph_default"
- - alert: "HardwareFanError"
+ - alert: "DIMMTemperatureHigh"
annotations:
- description: "Fan error(s) detected. Check `ceph health detail`."
- summary: "Fan error(s) detected on cluster {{ $labels.cluster }}"
- expr: "ceph_health_detail{name=\"HARDWARE_FANS\"} > 0"
- for: "30s"
+ description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
+ summary: "DIMM sensor {{ $labels.sensor_name }} above 80°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*DIMM.*_TEMP\"} > 80"
+ for: "1m"
labels:
- oid: "1.3.6.1.4.1.50495.1.2.1.13.6"
- severity: "critical"
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "ProcessorTemperatureHigh"
+ annotations:
+ description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
+ summary: "CPU sensor {{ $labels.sensor_name }} above 80°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*CPU_TEMP\"} > 80"
+ for: "1m"
+ labels:
+ severity: "warning"
+ type: "ceph_default"
+ - alert: "NVMeTemperatureHigh"
+ annotations:
+ description: "An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise."
+ summary: "NVMe sensor {{ $labels.sensor_name }} above 70°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
+ expr: "ceph_hardware_temperature_celsius{sensor_name=~\"NVME.*_TEMP\"} > 70"
+ for: "1m"
+ labels:
+ severity: "warning"
type: "ceph_default"
- name: "PrometheusServer"
rules:
summary: "The replication network usage on cluster mycluster has been increased over 80% in the last 30 minutes. Review the number of images being replicated. This alert will be cleaned automatically after 30 minutes"
description: "Detected a heavy increase in bandwidth for rbd replications (over 80%) in the last 30 min. This might not be a problem, but it is good to review the number of images being replicated simultaneously"
- - interval: 30s
+ # Power supply degraded
+ - interval: 1m
input_series:
- - series: 'ceph_health_detail{name="HARDWARE_STORAGE", cluster="mycluster"}'
- values: '1+0x40'
+ - series: 'ceph_hardware_health{category="power",component="PSU0",hostname="node1",cluster="mycluster"}'
+ values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- - expr: ceph_health_detail{name="HARDWARE_STORAGE"} > 0
+ - expr: ceph_hardware_health{category="power"} == 1
eval_time: 2m
exp_samples:
- - labels: '{__name__="ceph_health_detail", name="HARDWARE_STORAGE", cluster="mycluster"}'
+ - labels: 'ceph_hardware_health{category="power",component="PSU0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
- alertname: HardwareStorageError
+ alertname: PowerSupplyDegraded
- eval_time: 5m
- alertname: HardwareStorageError
+ alertname: PowerSupplyDegraded
exp_alerts:
- exp_labels:
- name: HARDWARE_STORAGE
+ category: power
+ component: PSU0
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "Power supply PSU0 degraded on node1 on cluster mycluster"
+ description: "PSU PSU0 is in a degraded state on node1."
+ # Power supply failed
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_health{category="power",component="PSU1",hostname="node2",cluster="mycluster"}'
+ values: '2 2 2 2 2 2 2 2 2'
+ promql_expr_test:
+ - expr: ceph_hardware_health{category="power"} == 2
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_health{category="power",component="PSU1",hostname="node2",cluster="mycluster"}'
+ value: 2
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: PowerSupplyFailed
+ - eval_time: 5m
+ alertname: PowerSupplyFailed
+ exp_alerts:
+ - exp_labels:
+ category: power
+ component: PSU1
+ hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
- oid: 1.3.6.1.4.1.50495.1.2.1.13.1
+ oid: 1.3.6.1.4.1.50495.1.2.1.13.5
exp_annotations:
- summary: Storage devices error(s) detected on cluster mycluster
- description: "Some storage devices are in error. Check `ceph health detail`."
- - interval: 30s
+ summary: "Power supply PSU1 failed on node2 on cluster mycluster"
+ description: "PSU fault detected on node2. PSU PSU1 has failed."
+ # Cooling degraded
+ - interval: 1m
input_series:
- - series: 'ceph_health_detail{name="HARDWARE_MEMORY", cluster="mycluster"}'
- values: '1+0x40'
+ - series: 'ceph_hardware_health{category="fans",component="FAN0",hostname="node1",cluster="mycluster"}'
+ values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- - expr: ceph_health_detail{name="HARDWARE_MEMORY"} > 0
+ - expr: ceph_hardware_health{category="fans"} == 1
eval_time: 2m
exp_samples:
- - labels: '{__name__="ceph_health_detail", name="HARDWARE_MEMORY", cluster="mycluster"}'
+ - labels: 'ceph_hardware_health{category="fans",component="FAN0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
- alertname: HardwareMemoryError
+ alertname: CoolingDegraded
- eval_time: 5m
- alertname: HardwareMemoryError
+ alertname: CoolingDegraded
exp_alerts:
- exp_labels:
- name: HARDWARE_MEMORY
+ category: fans
+ component: FAN0
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "Fan FAN0 degraded on node1 on cluster mycluster"
+ description: "Fan module FAN0 is in a degraded state on node1."
+ # Cooling failed
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_health{category="fans",component="FAN1",hostname="node2",cluster="mycluster"}'
+ values: '2 2 2 2 2 2 2 2 2'
+ promql_expr_test:
+ - expr: ceph_hardware_health{category="fans"} == 2
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_health{category="fans",component="FAN1",hostname="node2",cluster="mycluster"}'
+ value: 2
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: CoolingFailed
+ - eval_time: 5m
+ alertname: CoolingFailed
+ exp_alerts:
+ - exp_labels:
+ category: fans
+ component: FAN1
+ hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
- oid: 1.3.6.1.4.1.50495.1.2.1.13.2
+ oid: 1.3.6.1.4.1.50495.1.2.1.13.6
exp_annotations:
- summary: DIMM error(s) detected on cluster mycluster
- description: "DIMM error(s) detected. Check `ceph health detail`."
- - interval: 30s
+ summary: "Fan FAN1 failed on node2 on cluster mycluster"
+ description: "A cooling fault has been detected on node2. A cooling FAN within the enclosure has failed."
+ # NVMe drive degraded
+ - interval: 1m
input_series:
- - series: 'ceph_health_detail{name="HARDWARE_PROCESSOR", cluster="mycluster"}'
- values: '1+0x40'
+ - series: 'ceph_hardware_health{category="storage",component="nvme0n1",hostname="node1",cluster="mycluster"}'
+ values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- - expr: ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0
+ - expr: ceph_hardware_health{category="storage"} == 1
eval_time: 2m
exp_samples:
- - labels: '{__name__="ceph_health_detail", name="HARDWARE_PROCESSOR", cluster="mycluster"}'
+ - labels: 'ceph_hardware_health{category="storage",component="nvme0n1",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
- alertname: HardwareProcessorError
+ alertname: NVMeDriveDegraded
- eval_time: 5m
- alertname: HardwareProcessorError
+ alertname: NVMeDriveDegraded
exp_alerts:
- exp_labels:
- name: HARDWARE_PROCESSOR
+ category: storage
+ component: nvme0n1
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "Storage device nvme0n1 degraded on node1 on cluster mycluster"
+ description: "Storage device nvme0n1 is in a degraded state on node1."
+ # NVMe drive failed
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_health{category="storage",component="nvme1n1",hostname="node2",cluster="mycluster"}'
+ values: '2 2 2 2 2 2 2 2 2'
+ promql_expr_test:
+ - expr: ceph_hardware_health{category="storage"} == 2
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_health{category="storage",component="nvme1n1",hostname="node2",cluster="mycluster"}'
+ value: 2
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: NVMeDriveFailed
+ - eval_time: 5m
+ alertname: NVMeDriveFailed
+ exp_alerts:
+ - exp_labels:
+ category: storage
+ component: nvme1n1
+ hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
- oid: 1.3.6.1.4.1.50495.1.2.1.13.3
+ oid: 1.3.6.1.4.1.50495.1.2.1.13.1
exp_annotations:
- summary: Processor error(s) detected on cluster mycluster
- description: "Processor error(s) detected. Check `ceph health detail`."
- - interval: 30s
+ summary: "Storage device nvme1n1 failed on node2 on cluster mycluster"
+ description: "A drive fault has been detected on node2. Storage device nvme1n1 has failed."
+ # Memory degraded
+ - interval: 1m
input_series:
- - series: 'ceph_health_detail{name="HARDWARE_NETWORK", cluster="mycluster"}'
- values: '1+0x40'
+ - series: 'ceph_hardware_health{category="memory",component="DIMM_A0",hostname="node1",cluster="mycluster"}'
+ values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- - expr: ceph_health_detail{name="HARDWARE_NETWORK"} > 0
+ - expr: ceph_hardware_health{category="memory"} == 1
eval_time: 2m
exp_samples:
- - labels: '{__name__="ceph_health_detail", name="HARDWARE_NETWORK", cluster="mycluster"}'
+ - labels: 'ceph_hardware_health{category="memory",component="DIMM_A0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
- alertname: HardwareNetworkError
+ alertname: MemoryDegraded
+ - eval_time: 5m
+ alertname: MemoryDegraded
+ exp_alerts:
+ - exp_labels:
+ category: memory
+ component: DIMM_A0
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "Memory module DIMM_A0 degraded on node1 on cluster mycluster"
+ description: "Memory module DIMM_A0 is in a degraded state on node1."
+ # Memory failed
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_health{category="memory",component="DIMM_B0",hostname="node2",cluster="mycluster"}'
+ values: '2 2 2 2 2 2 2 2 2'
+ promql_expr_test:
+ - expr: ceph_hardware_health{category="memory"} == 2
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_health{category="memory",component="DIMM_B0",hostname="node2",cluster="mycluster"}'
+ value: 2
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: MemoryFailed
- eval_time: 5m
- alertname: HardwareNetworkError
+ alertname: MemoryFailed
exp_alerts:
- exp_labels:
- name: HARDWARE_NETWORK
+ category: memory
+ component: DIMM_B0
+ hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
- oid: 1.3.6.1.4.1.50495.1.2.1.13.4
+ oid: 1.3.6.1.4.1.50495.1.2.1.13.2
exp_annotations:
- summary: Network error(s) detected on cluster mycluster
- description: "Network error(s) detected. Check `ceph health detail`."
- - interval: 30s
+ summary: "Memory module DIMM_B0 failed on node2 on cluster mycluster"
+ description: "A memory fault has been detected on node2. Memory module DIMM_B0 has failed."
+ # Processor degraded
+ - interval: 1m
input_series:
- - series: 'ceph_health_detail{name="HARDWARE_POWER", cluster="mycluster"}'
- values: '1+0x40'
+ - series: 'ceph_hardware_health{category="processors",component="CPU0",hostname="node1",cluster="mycluster"}'
+ values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- - expr: ceph_health_detail{name="HARDWARE_POWER"} > 0
+ - expr: ceph_hardware_health{category="processors"} == 1
eval_time: 2m
exp_samples:
- - labels: '{__name__="ceph_health_detail", name="HARDWARE_POWER", cluster="mycluster"}'
+ - labels: 'ceph_hardware_health{category="processors",component="CPU0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
- alertname: HardwarePowerError
+ alertname: ProcessorDegraded
+ - eval_time: 5m
+ alertname: ProcessorDegraded
+ exp_alerts:
+ - exp_labels:
+ category: processors
+ component: CPU0
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "Processor CPU0 degraded on node1 on cluster mycluster"
+ description: "Processor CPU0 is in a degraded state on node1."
+ # Processor failed
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_health{category="processors",component="CPU1",hostname="node2",cluster="mycluster"}'
+ values: '2 2 2 2 2 2 2 2 2'
+ promql_expr_test:
+ - expr: ceph_hardware_health{category="processors"} == 2
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_health{category="processors",component="CPU1",hostname="node2",cluster="mycluster"}'
+ value: 2
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: ProcessorFailed
- eval_time: 5m
- alertname: HardwarePowerError
+ alertname: ProcessorFailed
exp_alerts:
- exp_labels:
- name: HARDWARE_POWER
+ category: processors
+ component: CPU1
+ hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
- oid: 1.3.6.1.4.1.50495.1.2.1.13.5
+ oid: 1.3.6.1.4.1.50495.1.2.1.13.3
exp_annotations:
- summary: Power supply error(s) detected on cluster mycluster
- description: "Power supply error(s) detected. Check `ceph health detail`."
- - interval: 30s
+ summary: "Processor CPU1 failed on node2 on cluster mycluster"
+ description: "A processor fault has been detected on node2. Processor CPU1 has failed."
+ # Network degraded
+ - interval: 1m
input_series:
- - series: 'ceph_health_detail{name="HARDWARE_FANS", cluster="mycluster"}'
- values: '1+0x40'
+ - series: 'ceph_hardware_health{category="network",component="eth0",hostname="node1",cluster="mycluster"}'
+ values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- - expr: ceph_health_detail{name="HARDWARE_FANS"} > 0
+ - expr: ceph_hardware_health{category="network"} == 1
eval_time: 2m
exp_samples:
- - labels: '{__name__="ceph_health_detail", name="HARDWARE_FANS", cluster="mycluster"}'
+ - labels: 'ceph_hardware_health{category="network",component="eth0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
- alertname: HardwareFanError
+ alertname: NetworkDegraded
+ - eval_time: 5m
+ alertname: NetworkDegraded
+ exp_alerts:
+ - exp_labels:
+ category: network
+ component: eth0
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "Network interface eth0 degraded on node1 on cluster mycluster"
+ description: "Network interface eth0 is in a degraded state on node1."
+ # Network failed
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_health{category="network",component="eth1",hostname="node2",cluster="mycluster"}'
+ values: '2 2 2 2 2 2 2 2 2'
+ promql_expr_test:
+ - expr: ceph_hardware_health{category="network"} == 2
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_health{category="network",component="eth1",hostname="node2",cluster="mycluster"}'
+ value: 2
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: NetworkFailed
- eval_time: 5m
- alertname: HardwareFanError
+ alertname: NetworkFailed
exp_alerts:
- exp_labels:
- name: HARDWARE_FANS
+ category: network
+ component: eth1
+ hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
- oid: 1.3.6.1.4.1.50495.1.2.1.13.6
+ oid: 1.3.6.1.4.1.50495.1.2.1.13.4
+ exp_annotations:
+ summary: "Network interface eth1 failed on node2 on cluster mycluster"
+ description: "A network fault has been detected on node2. Network interface eth1 has failed."
+ # Motherboard temperature high
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_temperature_celsius{sensor_name="MB_TEMP1",hostname="node1",cluster="mycluster"}'
+ values: '65 65 65 65 65 65 65 65 65'
+ promql_expr_test:
+ - expr: ceph_hardware_temperature_celsius{sensor_name=~".*MB_TEMP.*"} > 60
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_temperature_celsius{sensor_name="MB_TEMP1",hostname="node1",cluster="mycluster"}'
+ value: 65
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: MotherboardTemperatureHigh
+ - eval_time: 5m
+ alertname: MotherboardTemperatureHigh
+ exp_alerts:
+ - exp_labels:
+ sensor_name: MB_TEMP1
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "Motherboard sensor MB_TEMP1 above 60°C on node1 on cluster mycluster"
+ description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
+ # DIMM temperature high
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_temperature_celsius{sensor_name="DIMM_A0_TEMP",hostname="node1",cluster="mycluster"}'
+ values: '85 85 85 85 85 85 85 85 85'
+ promql_expr_test:
+ - expr: ceph_hardware_temperature_celsius{sensor_name=~".*DIMM.*_TEMP"} > 80
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_temperature_celsius{sensor_name="DIMM_A0_TEMP",hostname="node1",cluster="mycluster"}'
+ value: 85
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: DIMMTemperatureHigh
+ - eval_time: 5m
+ alertname: DIMMTemperatureHigh
+ exp_alerts:
+ - exp_labels:
+ sensor_name: DIMM_A0_TEMP
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "DIMM sensor DIMM_A0_TEMP above 80°C on node1 on cluster mycluster"
+ description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
+ # Processor temperature high
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_temperature_celsius{sensor_name="CPU_TEMP",hostname="node1",cluster="mycluster"}'
+ values: '85 85 85 85 85 85 85 85 85'
+ promql_expr_test:
+ - expr: ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"} > 80
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_temperature_celsius{sensor_name="CPU_TEMP",hostname="node1",cluster="mycluster"}'
+ value: 85
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: ProcessorTemperatureHigh
+ - eval_time: 5m
+ alertname: ProcessorTemperatureHigh
+ exp_alerts:
+ - exp_labels:
+ sensor_name: CPU_TEMP
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
+ exp_annotations:
+ summary: "CPU sensor CPU_TEMP above 80°C on node1 on cluster mycluster"
+ description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
+ # NVMe temperature high
+ - interval: 1m
+ input_series:
+ - series: 'ceph_hardware_temperature_celsius{sensor_name="NVME0_TEMP",hostname="node1",cluster="mycluster"}'
+ values: '75 75 75 75 75 75 75 75 75'
+ promql_expr_test:
+ - expr: ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"} > 70
+ eval_time: 2m
+ exp_samples:
+ - labels: 'ceph_hardware_temperature_celsius{sensor_name="NVME0_TEMP",hostname="node1",cluster="mycluster"}'
+ value: 75
+ alert_rule_test:
+ - eval_time: 1m
+ alertname: NVMeTemperatureHigh
+ - eval_time: 5m
+ alertname: NVMeTemperatureHigh
+ exp_alerts:
+ - exp_labels:
+ sensor_name: NVME0_TEMP
+ hostname: node1
+ severity: warning
+ cluster: mycluster
+ type: ceph_default
exp_annotations:
- summary: Fan error(s) detected on cluster mycluster
- description: "Fan error(s) detected. Check `ceph health detail`."
+ summary: "NVMe sensor NVME0_TEMP above 70°C on node1 on cluster mycluster"
+ description: "An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise."
# nvmeof Tests
# NVMeoFSubsystemNamespaceLimit
projects / ceph.git / commitdiff