mgr/dashboard: enable addition custom Prometheus alerts

Fixes: https://tracker.ceph.com/issues/57294
Signed-off-by: Patrick Seidensal <pseidensal@suse.com>
(cherry picked from commit 3551d7f8b36d883a72b85f0bd5568a33ac00e62c)

Conflicts:
	doc/cephadm/services/monitoring.rst
	src/pybind/mgr/cephadm/services/monitoring.py
	src/pybind/mgr/cephadm/tests/test_services.py

diff --git a/doc/cephadm/services/monitoring.rst b/doc/cephadm/services/monitoring.rst
index 5f9d828909579c06812fd85d8950315dbf3b2605..08ccd94821e081006b6aaf503482531ed812c0cc 100644 (file)
--- a/doc/cephadm/services/monitoring.rst
+++ b/doc/cephadm/services/monitoring.rst
@@ -217,6 +217,7 @@ set``:
 - ``services/grafana/ceph-dashboard.yml``
 - ``services/grafana/grafana.ini``
 - ``services/prometheus/prometheus.yml``
+- ``services/prometheus/alerting/custom_alerts.yml``
 
 You can look up the file templates that are currently used by cephadm in
 ``src/pybind/mgr/cephadm/templates``:
@@ -262,6 +263,15 @@ Example
   # reconfig the prometheus service
   ceph orch reconfig prometheus
 
+.. code-block:: bash
+
+  # set additional custom alerting rules for Prometheus
+  ceph config-key set mgr/cephadm/services/prometheus/alerting/custom_alerts.yml \
+    -i $PWD/custom_alerts.yml
+
+  # Note that custom alerting rules are not parsed by Jinja and hence escaping
+  # will not be an issue.
+
 Deploying monitoring without cephadm
 ------------------------------------
 

diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py
index d07c67bd5b979f5b4650d8344416a09b26258f6b..6e4bfd73e7117b9696116e1c9c976349ddf259e6 100644 (file)
--- a/src/pybind/mgr/cephadm/services/monitoring.py
+++ b/src/pybind/mgr/cephadm/services/monitoring.py
@@ -360,6 +360,23 @@ class PrometheusService(CephadmService):
                 alerts = f.read()
             r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts
 
+        # Include custom alerts if present in key value store. This enables the
+        # users to add custom alerts. Write the file in any case, so that if the
+        # content of the key value store changed, that file is overwritten
+        # (emptied in case they value has been removed from the key value
+        # store). This prevents the necessity to adapt `cephadm` binary to
+        # remove the file.
+        #
+        # Don't use the template engine for it as
+        #
+        #   1. the alerts are always static and
+        #   2. they are a template themselves for the Go template engine, which
+        #      use curly braces and escaping that is cumbersome and unnecessary
+        #      for the user.
+        #
+        r['files']['/etc/prometheus/alerting/custom_alerts.yml'] = \
+            self.mgr.get_store('services/prometheus/alerting/custom_alerts.yml', '')
+
         return r, sorted(deps)
 
     def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription:

diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index 102d6e7cc8055a183a7fe262efc9b1f8c9237437..a8cb392348a68241ea121f3f3ce45ae3ebebf489 100644 (file)
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -426,7 +426,8 @@ class TestMonitoring:
                         '--config-json', '-',
                         '--tcp-ports', '9095'
                     ],
-                    stdin=json.dumps({"files": {"prometheus.yml": y}}),
+                    stdin=json.dumps({"files": {"prometheus.yml": y,
+                                                "/etc/prometheus/alerting/custom_alerts.yml": ""}}),
                     image='')
 
     @patch("cephadm.serve.CephadmServe._run_cephadm")