Resize whisper databases when necessary

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/ansible/roles/ceph-grafana/handlers/main.yml b/ansible/roles/ceph-grafana/handlers/main.yml
index 7aa2c61bed0abb12509c5a9d9965d8ae3be174d1..a14a7bdede159748c1064c41b1e34fdb3385cae6 100644 (file)
--- a/ansible/roles/ceph-grafana/handlers/main.yml
+++ b/ansible/roles/ceph-grafana/handlers/main.yml
@@ -5,6 +5,13 @@
     state: restarted
     enabled: true
 
+- name: Resize whisper databases
+  # xargs -P parallelizes execution; scale it to the number of cores on the system
+  shell: "find /var/lib/carbon/whisper -name '*.wsp' -print0 | xargs -n1 -0 -P {{ ansible_processor_vcpus }} -I {} whisper-resize {} {{ whisper_retention.replace(',', ' ') }}"
+  register: whisper_resize
+  failed_when: "'Traceback' in whisper_resize.stdout"
+  no_log: true
+
 - name: Restart graphite-web
   service:
     # graphite-web is served by httpd

diff --git a/ansible/roles/ceph-grafana/tasks/configure_carbon.yml b/ansible/roles/ceph-grafana/tasks/configure_carbon.yml
index 8b84c06b45accfb24d86e47b914b51ed47c731f7..79c292da18e3cb589ff5ee8d63f886c8c3e461ad 100644 (file)
--- a/ansible/roles/ceph-grafana/tasks/configure_carbon.yml
+++ b/ansible/roles/ceph-grafana/tasks/configure_carbon.yml
@@ -1,14 +1,12 @@
 ---
+- set_fact:
+    # Changing collectd_interval is not supported at this time
+    whisper_retention: "{{ collectd_interval|default('10') }}s:7d,1m:30d,15m:5y"
+
 - name: Configure retention for collectd stats
-  blockinfile:
+  template:
+    src: storage-schemas.conf
     dest: /etc/carbon/storage-schemas.conf
-    # NOTE: Retention settings are applied to metrics as they are *created*. If
-    # these settings are changed after-the-fact, it's necessary to either
-    # delete the whisper files (!) or run whisper-resize on them all.
-    block: |
-      [collectd]
-      pattern = ^collectd\.
-      retentions = 1s:7d,1m:30d,15m:5y
-    # This must be above other declarations in the file
-    insertbefore: "BOF"
-  notify: Restart carbon-cache
+  notify:
+    - Resize whisper databases
+    - Restart carbon-cache

diff --git a/ansible/roles/ceph-grafana/templates/storage-schemas.conf b/ansible/roles/ceph-grafana/templates/storage-schemas.conf
new file mode 100644 (file)
index 0000000..5237b32
--- /dev/null
+++ b/ansible/roles/ceph-grafana/templates/storage-schemas.conf
@@ -0,0 +1,19 @@
+# Schema definitions for Whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds.
+#
+#  [name]
+#  pattern = regex
+#  retentions = timePerPoint:timeToStore, timePerPoint:timeToStore, ...
+[collectd]
+pattern = ^collectd\.
+retentions = {{ whisper_retention }}
+
+# Carbon's internal metrics. This entry should match what is specified in
+# CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings
+[carbon]
+pattern = ^carbon\.
+retentions = 60:90d
+
+[default_1min_for_1day]
+pattern = .*
+retentions = 60s:1d