options, Mon: monitor the snap trim queues

If new option "mon osd snap trim queue warn on" is set to value larger
than 0 (32768 by default), cluster will go into HEALTH_WARN state
once any pg has a snap trim queue larger than that value. This can
be used as an indicator of snaptrimmer not keeping up and disk space
not being reclaimed fast enough. Warning message will tell how many
pgs are affected.

Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
(cherry picked from commit 8412a65e0e6f610fb39430d8f65e561b6dbda13f)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index 1168ba78344cfc18003595f48cac0fb53c4033fe..a36122f561c883dbc4669de471bd8a601732e1e3 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -291,6 +291,7 @@ OPTION(mon_osd_reporter_subtree_level , OPT_STR)   // in which level of parent b
 OPTION(mon_osd_force_trim_to, OPT_INT)   // force mon to trim maps to this point, regardless of min_last_epoch_clean (dangerous)
 OPTION(mon_mds_force_trim_to, OPT_INT)   // force mon to trim mdsmaps to this point (dangerous)
 OPTION(mon_mds_skip_sanity, OPT_BOOL)  // skip safety assertions on FSMap (in case of bugs where we want to continue anyway)
+OPTION(mon_osd_snap_trim_queue_warn_on, OPT_INT)
 
 // monitor debug options
 OPTION(mon_debug_deprecated_as_obsolete, OPT_BOOL) // consider deprecated commands as obsolete

diff --git a/src/common/options.cc b/src/common/options.cc
index 90098d82322c4d983791f5acc121d099377996f1..fb7a1ad6d022db06685e1da63a8eeac006df8ea0 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1294,6 +1294,11 @@ std::vector<Option> get_global_options() {
     .set_default("host")
     .set_description(""),
 
+    Option("mon_osd_snap_trim_queue_warn_on", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(32768)
+    .set_description("Warn when snap trim queue is that large (or larger).")
+    .set_long_description("Warn when snap trim queue length for at least one PG crosses this value, as this is indicator of snap trimmer not keeping up, wasting disk space"),
+
     Option("mon_osd_force_trim_to", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description(""),

diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 834da03debc922969051ca8a1a03211075b28f09..e06afb4d640dea46fea014abb8c4c0608b08da4b 100644 (file)
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -3307,6 +3307,50 @@ void PGMap::get_health_checks(
       d.detail.swap(detail);
     }
   }
+
+  // PG_SLOW_SNAP_TRIMMING
+  if (!pg_stat.empty() && cct->_conf->mon_osd_snap_trim_queue_warn_on > 0) {
+    uint32_t snapthreshold = cct->_conf->mon_osd_snap_trim_queue_warn_on;
+    uint64_t snaptrimq_exceeded = 0;
+    uint32_t longest_queue = 0;
+    const pg_t* longest_q_pg = nullptr;
+    list<string> detail;
+
+    for (auto& i: pg_stat) {
+      uint32_t current_len = i.second.snaptrimq_len;
+      if (current_len >= snapthreshold) {
+        snaptrimq_exceeded++;
+        if (longest_queue <= current_len) {
+          longest_q_pg = &i.first;
+          longest_queue = current_len;
+        }
+        if (detail.size() < max - 1) {
+          stringstream ss;
+          ss << "snap trim queue for pg " << i.first << " at " << current_len;
+          detail.push_back(ss.str());
+          continue;
+        }
+        if (detail.size() < max) {
+          detail.push_back("...more pgs affected");
+          continue;
+        }
+      }
+    }
+
+    if (snaptrimq_exceeded) {
+      {
+         ostringstream ss;
+         ss << "longest queue on pg " << *longest_q_pg << " at " << longest_queue;
+         detail.push_back(ss.str());
+      }
+
+      stringstream ss;
+      ss << "snap trim queue for " << snaptrimq_exceeded << " pg(s) >= " << snapthreshold << " (mon_osd_snap_trim_queue_warn_on)";
+      auto& d = checks->add("PG_SLOW_SNAP_TRIMMING", HEALTH_WARN, ss.str());
+      detail.push_back("try decreasing \"osd snap trim sleep\" and/or increasing \"osd pg max concurrent snap trims\".");
+      d.detail.swap(detail);
+    }
+  }
 }
 
 void PGMap::get_health(