From 8412a65e0e6f610fb39430d8f65e561b6dbda13f Mon Sep 17 00:00:00 2001
From: =?utf8?q?Piotr=20Da=C5=82ek?= <piotr.dalek@corp.ovh.com>
Date: Wed, 13 Dec 2017 16:13:33 +0100
Subject: [PATCH] options, Mon: monitor the snap trim queues
MIME-Version: 1.0
Content-Type: text/plain; charset=utf8
Content-Transfer-Encoding: 8bit

If new option "mon osd snap trim queue warn on" is set to value larger
than 0 (32768 by default), cluster will go into HEALTH_WARN state
once any pg has a snap trim queue larger than that value. This can
be used as an indicator of snaptrimmer not keeping up and disk space
not being reclaimed fast enough. Warning message will tell how many
pgs are affected.

Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
---
 src/common/legacy_config_opts.h |  1 +
 src/common/options.cc           |  5 ++++
 src/mon/PGMap.cc                | 44 +++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index c82649836f8..c5c0d6d8cd4 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -286,6 +286,7 @@ OPTION(mon_inject_sync_get_chunk_delay, OPT_DOUBLE)  // inject N second delay on
 OPTION(mon_osd_force_trim_to, OPT_INT)   // force mon to trim maps to this point, regardless of min_last_epoch_clean (dangerous)
 OPTION(mon_mds_force_trim_to, OPT_INT)   // force mon to trim mdsmaps to this point (dangerous)
 OPTION(mon_mds_skip_sanity, OPT_BOOL)  // skip safety assertions on FSMap (in case of bugs where we want to continue anyway)
+OPTION(mon_osd_snap_trim_queue_warn_on, OPT_INT)
 
 // monitor debug options
 OPTION(mon_debug_deprecated_as_obsolete, OPT_BOOL) // consider deprecated commands as obsolete
diff --git a/src/common/options.cc b/src/common/options.cc
index 245240efed8..cb45f68509b 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1324,6 +1324,11 @@ std::vector<Option> get_global_options() {
     .set_safe()
     .set_description("in which level of parent bucket the reporters are counted"),
 
+    Option("mon_osd_snap_trim_queue_warn_on", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(32768)
+    .set_description("Warn when snap trim queue is that large (or larger).")
+    .set_long_description("Warn when snap trim queue length for at least one PG crosses this value, as this is indicator of snap trimmer not keeping up, wasting disk space"),
+
     Option("mon_osd_force_trim_to", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description(""),
diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 4afa37826a9..f92bc96c583 100644
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -2875,6 +2875,50 @@ void PGMap::get_health_checks(
       d.detail.swap(detail);
     }
   }
+
+  // PG_SLOW_SNAP_TRIMMING
+  if (!pg_stat.empty() && cct->_conf->mon_osd_snap_trim_queue_warn_on > 0) {
+    uint32_t snapthreshold = cct->_conf->mon_osd_snap_trim_queue_warn_on;
+    uint64_t snaptrimq_exceeded = 0;
+    uint32_t longest_queue = 0;
+    const pg_t* longest_q_pg = nullptr;
+    list<string> detail;
+
+    for (auto& i: pg_stat) {
+      uint32_t current_len = i.second.snaptrimq_len;
+      if (current_len >= snapthreshold) {
+        snaptrimq_exceeded++;
+        if (longest_queue <= current_len) {
+          longest_q_pg = &i.first;
+          longest_queue = current_len;
+        }
+        if (detail.size() < max - 1) {
+          stringstream ss;
+          ss << "snap trim queue for pg " << i.first << " at " << current_len;
+          detail.push_back(ss.str());
+          continue;
+        }
+        if (detail.size() < max) {
+          detail.push_back("...more pgs affected");
+          continue;
+        }
+      }
+    }
+
+    if (snaptrimq_exceeded) {
+      {
+         ostringstream ss;
+         ss << "longest queue on pg " << *longest_q_pg << " at " << longest_queue;
+         detail.push_back(ss.str());
+      }
+
+      stringstream ss;
+      ss << "snap trim queue for " << snaptrimq_exceeded << " pg(s) >= " << snapthreshold << " (mon_osd_snap_trim_queue_warn_on)";
+      auto& d = checks->add("PG_SLOW_SNAP_TRIMMING", HEALTH_WARN, ss.str());
+      detail.push_back("try decreasing \"osd snap trim sleep\" and/or increasing \"osd pg max concurrent snap trims\".");
+      d.detail.swap(detail);
+    }
+  }
 }
 
 int process_pg_map_command(
-- 
2.39.5