From 768a58814ebdcae62e67c2045fd19bce6687a271 Mon Sep 17 00:00:00 2001 From: Patrick Seidensal Date: Thu, 25 Aug 2022 14:47:52 +0200 Subject: [PATCH] mgr/dashboard: enable addition custom Prometheus alerts Fixes: https://tracker.ceph.com/issues/57294 Signed-off-by: Patrick Seidensal (cherry picked from commit 3551d7f8b36d883a72b85f0bd5568a33ac00e62c) Conflicts: doc/cephadm/services/monitoring.rst src/pybind/mgr/cephadm/services/monitoring.py src/pybind/mgr/cephadm/tests/test_services.py --- doc/cephadm/services/monitoring.rst | 10 ++++++++++ src/pybind/mgr/cephadm/services/monitoring.py | 17 +++++++++++++++++ src/pybind/mgr/cephadm/tests/test_services.py | 3 ++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/doc/cephadm/services/monitoring.rst b/doc/cephadm/services/monitoring.rst index 5f9d828909579..08ccd94821e08 100644 --- a/doc/cephadm/services/monitoring.rst +++ b/doc/cephadm/services/monitoring.rst @@ -217,6 +217,7 @@ set``: - ``services/grafana/ceph-dashboard.yml`` - ``services/grafana/grafana.ini`` - ``services/prometheus/prometheus.yml`` +- ``services/prometheus/alerting/custom_alerts.yml`` You can look up the file templates that are currently used by cephadm in ``src/pybind/mgr/cephadm/templates``: @@ -262,6 +263,15 @@ Example # reconfig the prometheus service ceph orch reconfig prometheus +.. code-block:: bash + + # set additional custom alerting rules for Prometheus + ceph config-key set mgr/cephadm/services/prometheus/alerting/custom_alerts.yml \ + -i $PWD/custom_alerts.yml + + # Note that custom alerting rules are not parsed by Jinja and hence escaping + # will not be an issue. + Deploying monitoring without cephadm ------------------------------------ diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py index d07c67bd5b979..6e4bfd73e7117 100644 --- a/src/pybind/mgr/cephadm/services/monitoring.py +++ b/src/pybind/mgr/cephadm/services/monitoring.py @@ -360,6 +360,23 @@ class PrometheusService(CephadmService): alerts = f.read() r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts + # Include custom alerts if present in key value store. This enables the + # users to add custom alerts. Write the file in any case, so that if the + # content of the key value store changed, that file is overwritten + # (emptied in case they value has been removed from the key value + # store). This prevents the necessity to adapt `cephadm` binary to + # remove the file. + # + # Don't use the template engine for it as + # + # 1. the alerts are always static and + # 2. they are a template themselves for the Go template engine, which + # use curly braces and escaping that is cumbersome and unnecessary + # for the user. + # + r['files']['/etc/prometheus/alerting/custom_alerts.yml'] = \ + self.mgr.get_store('services/prometheus/alerting/custom_alerts.yml', '') + return r, sorted(deps) def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription: diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py index 102d6e7cc8055..a8cb392348a68 100644 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ b/src/pybind/mgr/cephadm/tests/test_services.py @@ -426,7 +426,8 @@ class TestMonitoring: '--config-json', '-', '--tcp-ports', '9095' ], - stdin=json.dumps({"files": {"prometheus.yml": y}}), + stdin=json.dumps({"files": {"prometheus.yml": y, + "/etc/prometheus/alerting/custom_alerts.yml": ""}}), image='') @patch("cephadm.serve.CephadmServe._run_cephadm") -- 2.39.5