From e5e2accab9eded103f8146a5f4273455112dafb6 Mon Sep 17 00:00:00 2001 From: Patrick Seidensal Date: Thu, 25 Aug 2022 14:47:52 +0200 Subject: [PATCH] mgr/dashboard: enable addition custom Prometheus alerts Fixes: https://tracker.ceph.com/issues/57294 Signed-off-by: Patrick Seidensal (cherry picked from commit 3551d7f8b36d883a72b85f0bd5568a33ac00e62c) Conflicts: doc/cephadm/services/monitoring.rst src/pybind/mgr/cephadm/services/monitoring.py src/pybind/mgr/cephadm/tests/test_services.py --- doc/cephadm/services/monitoring.rst | 10 ++++++++++ src/pybind/mgr/cephadm/services/monitoring.py | 17 +++++++++++++++++ src/pybind/mgr/cephadm/tests/test_services.py | 4 +++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/doc/cephadm/services/monitoring.rst b/doc/cephadm/services/monitoring.rst index a36f62edeec7..6c332befe886 100644 --- a/doc/cephadm/services/monitoring.rst +++ b/doc/cephadm/services/monitoring.rst @@ -195,6 +195,7 @@ set``: - ``services/grafana/ceph-dashboard.yml`` - ``services/grafana/grafana.ini`` - ``services/prometheus/prometheus.yml`` +- ``services/prometheus/alerting/custom_alerts.yml`` You can look up the file templates that are currently used by cephadm in ``src/pybind/mgr/cephadm/templates``: @@ -240,6 +241,15 @@ Example # reconfig the prometheus service ceph orch reconfig prometheus +.. code-block:: bash + + # set additional custom alerting rules for Prometheus + ceph config-key set mgr/cephadm/services/prometheus/alerting/custom_alerts.yml \ + -i $PWD/custom_alerts.yml + + # Note that custom alerting rules are not parsed by Jinja and hence escaping + # will not be an issue. + Deploying monitoring without cephadm ------------------------------------ diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py index e7ef15fd3055..8de7195a3ad9 100644 --- a/src/pybind/mgr/cephadm/services/monitoring.py +++ b/src/pybind/mgr/cephadm/services/monitoring.py @@ -375,6 +375,23 @@ class PrometheusService(CephadmService): alerts = f.read() r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts + # Include custom alerts if present in key value store. This enables the + # users to add custom alerts. Write the file in any case, so that if the + # content of the key value store changed, that file is overwritten + # (emptied in case they value has been removed from the key value + # store). This prevents the necessity to adapt `cephadm` binary to + # remove the file. + # + # Don't use the template engine for it as + # + # 1. the alerts are always static and + # 2. they are a template themselves for the Go template engine, which + # use curly braces and escaping that is cumbersome and unnecessary + # for the user. + # + r['files']['/etc/prometheus/alerting/custom_alerts.yml'] = \ + self.mgr.get_store('services/prometheus/alerting/custom_alerts.yml', '') + return r, sorted(deps) def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription: diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py index 876024c918a1..57cd12456aaf 100644 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ b/src/pybind/mgr/cephadm/tests/test_services.py @@ -425,7 +425,9 @@ class TestMonitoring: '--config-json', '-', '--tcp-ports', '9095' ], - stdin=json.dumps({"files": {"prometheus.yml": y}, 'retention_time': '15d'}), + stdin=json.dumps({"files": {"prometheus.yml": y, + "/etc/prometheus/alerting/custom_alerts.yml": ""}, + 'retention_time': '15d'}), image='') @patch("cephadm.serve.CephadmServe._run_cephadm") -- 2.47.3