monitoring: Fix NVMeoF subsys/namespace limit alerts

Change NVMeoFTooManyNamespaces and NVMeoFTooManySubsystems
alert to trigger for ">= $limit" instead of "> $limit".

Signed-off-by: Vallari Agrawal <vallari.agrawal@ibm.com>

diff --git a/monitoring/ceph-mixin/prometheus_alerts.libsonnet b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
index 3fb84e24206ec93dd496cac5d584f34450a1ba75..f32798093c0bb990e223f13430db5d5f132e5af9 100644 (file)
--- a/monitoring/ceph-mixin/prometheus_alerts.libsonnet
+++ b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
@@ -928,21 +928,21 @@
         {
           alert: 'NVMeoFTooManySubsystems',
           'for': '1m',
-          expr: 'count by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_metadata,"gateway_host","$1","instance","(.*?)(?::.*)?")) > %.2f' % [$._config.NVMeoFMaxSubsystemsPerGateway],
+          expr: 'count by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_metadata,"gateway_host","$1","instance","(.*?)(?::.*)?")) >= %.2f' % [$._config.NVMeoFMaxSubsystemsPerGateway],
           labels: { severity: 'warning', type: 'ceph_default' },
           annotations: {
-            summary: 'The number of subsystems defined to the gateway exceeds supported values%(cluster)s' % $.MultiClusterSummary(),
-            description: 'Although you may continue to create subsystems in {{ $labels.gateway_host }}, the configuration may not be supported',
+            summary: 'The number of subsystems defined to the NVMeoF gateway reached or exceeded the supported values%(cluster)s' % $.MultiClusterSummary(),
+            description: 'NVMeoF gateway {{ $labels.gateway_host }} has reached or exceeded the supported maximum of %(NVMeoFMaxSubsystemsPerGateway)d subsystems. Current count: {{ $value }}.' % $._config,
           },
         },
         {
           alert: 'NVMeoFTooManyNamespaces',
           'for': '1m',
-          expr: 'sum by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_namespace_count,"gateway_host","$1","instance","(.*?)(?::.*)?")) > %.2f' % [$._config.NVMeoFMaxNamespaces],
+          expr: 'sum by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_namespace_count,"gateway_host","$1","instance","(.*?)(?::.*)?")) >= %.2f' % [$._config.NVMeoFMaxNamespaces],
           labels: { severity: 'warning', type: 'ceph_default' },
           annotations: {
-            summary: 'The number of namespaces defined to the gateway exceeds supported values%(cluster)s' % $.MultiClusterSummary(),
-            description: 'Although you may continue to create namespaces in {{ $labels.gateway_host }}, the configuration may not be supported',
+            summary: 'The number of namespaces defined to the NVMeoF gateway reached or exceeded supported values%(cluster)s' % $.MultiClusterSummary(),
+            description: 'NVMeoF gateway {{ $labels.gateway_host }} has reached or exceeded the supported maximum of %(NVMeoFMaxNamespaces)d namespaces. Current count: {{ $value }}.' % $._config,
           },
         },
         {

diff --git a/monitoring/ceph-mixin/prometheus_alerts.yml b/monitoring/ceph-mixin/prometheus_alerts.yml
index 1ce17ac07cd857b123ac7c6cf2c2896ceb59236d..22d121e70764b9cf04a52ea8506b154a20ff7693 100644 (file)
--- a/monitoring/ceph-mixin/prometheus_alerts.yml
+++ b/monitoring/ceph-mixin/prometheus_alerts.yml
@@ -830,18 +830,18 @@ groups:
           type: "ceph_default"
       - alert: "NVMeoFTooManySubsystems"
         annotations:
-          description: "Although you may continue to create subsystems in {{ $labels.gateway_host }}, the configuration may not be supported"
-          summary: "The number of subsystems defined to the gateway exceeds supported values on cluster {{ $labels.cluster }}"
-        expr: "count by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_metadata,\"gateway_host\",\"$1\",\"instance\",\"(.*?)(?::.*)?\")) > 128.00"
+          description: "NVMeoF gateway {{ $labels.gateway_host }} has reached or exceeded the supported maximum of 128 subsystems. Current count: {{ $value }}."
+          summary: "The number of subsystems defined to the NVMeoF gateway reached or exceeded the supported values on cluster {{ $labels.cluster }}"
+        expr: "count by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_metadata,\"gateway_host\",\"$1\",\"instance\",\"(.*?)(?::.*)?\")) >= 128.00"
         for: "1m"
         labels:
           severity: "warning"
           type: "ceph_default"
       - alert: "NVMeoFTooManyNamespaces"
         annotations:
-          description: "Although you may continue to create namespaces in {{ $labels.gateway_host }}, the configuration may not be supported"
-          summary: "The number of namespaces defined to the gateway exceeds supported values on cluster {{ $labels.cluster }}"
-        expr: "sum by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_namespace_count,\"gateway_host\",\"$1\",\"instance\",\"(.*?)(?::.*)?\")) > 2048.00"
+          description: "NVMeoF gateway {{ $labels.gateway_host }} has reached or exceeded the supported maximum of 2048 namespaces. Current count: {{ $value }}."
+          summary: "The number of namespaces defined to the NVMeoF gateway reached or exceeded supported values on cluster {{ $labels.cluster }}"
+        expr: "sum by(gateway_host, cluster) (label_replace(ceph_nvmeof_subsystem_namespace_count,\"gateway_host\",\"$1\",\"instance\",\"(.*?)(?::.*)?\")) >= 2048.00"
         for: "1m"
         labels:
           severity: "warning"

diff --git a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
index 69e7b3a4b9cbc1aaf1376132e0a943609044c88c..1184aa084db1e3a87e7f7585124cb665ffdfe54c 100644 (file)
--- a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
+++ b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
@@ -2838,8 +2838,8 @@ tests:
           cluster: mycluster
           type: ceph_default
         exp_annotations:
-          summary: "The number of subsystems defined to the gateway exceeds supported values on cluster mycluster"
-          description: "Although you may continue to create subsystems in node-1, the configuration may not be supported"
+          summary: "The number of subsystems defined to the NVMeoF gateway reached or exceeded the supported values on cluster mycluster"
+          description: "NVMeoF gateway node-1 has reached or exceeded the supported maximum of 128 subsystems. Current count: 129."
 
 # NVMeoFTooManyNamespaces
  - interval: 1m
@@ -2882,8 +2882,8 @@ tests:
           cluster: mycluster
           type: ceph_default
         exp_annotations:
-          summary: "The number of namespaces defined to the gateway exceeds supported values on cluster mycluster"
-          description: "Although you may continue to create namespaces in node-1, the configuration may not be supported"
+          summary: "The number of namespaces defined to the NVMeoF gateway reached or exceeded supported values on cluster mycluster"
+          description: "NVMeoF gateway node-1 has reached or exceeded the supported maximum of 2048 namespaces. Current count: 2200."
 
  # NVMeoFVersionMismatch
  - interval: 1m