From: Piotr Dałek <piotr.dalek@corp.ovh.com>
Date: Wed, 13 Dec 2017 15:13:33 +0000 (+0100)
Subject: options, Mon: monitor the snap trim queues
X-Git-Tag: v13.0.2~572^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8412a65e0e6f610fb39430d8f65e561b6dbda13f;p=ceph.git

options, Mon: monitor the snap trim queues

If new option "mon osd snap trim queue warn on" is set to value larger
than 0 (32768 by default), cluster will go into HEALTH_WARN state
once any pg has a snap trim queue larger than that value. This can
be used as an indicator of snaptrimmer not keeping up and disk space
not being reclaimed fast enough. Warning message will tell how many
pgs are affected.

Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
---

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index c82649836f82..c5c0d6d8cd43 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -286,6 +286,7 @@ OPTION(mon_inject_sync_get_chunk_delay, OPT_DOUBLE)  // inject N second delay on
 OPTION(mon_osd_force_trim_to, OPT_INT)   // force mon to trim maps to this point, regardless of min_last_epoch_clean (dangerous)
 OPTION(mon_mds_force_trim_to, OPT_INT)   // force mon to trim mdsmaps to this point (dangerous)
 OPTION(mon_mds_skip_sanity, OPT_BOOL)  // skip safety assertions on FSMap (in case of bugs where we want to continue anyway)
+OPTION(mon_osd_snap_trim_queue_warn_on, OPT_INT)
 
 // monitor debug options
 OPTION(mon_debug_deprecated_as_obsolete, OPT_BOOL) // consider deprecated commands as obsolete
diff --git a/src/common/options.cc b/src/common/options.cc
index 245240efed8d..cb45f68509bb 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1324,6 +1324,11 @@ std::vector<Option> get_global_options() {
     .set_safe()
     .set_description("in which level of parent bucket the reporters are counted"),
 
+    Option("mon_osd_snap_trim_queue_warn_on", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(32768)
+    .set_description("Warn when snap trim queue is that large (or larger).")
+    .set_long_description("Warn when snap trim queue length for at least one PG crosses this value, as this is indicator of snap trimmer not keeping up, wasting disk space"),
+
     Option("mon_osd_force_trim_to", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description(""),
diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 4afa37826a9e..f92bc96c5834 100644
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -2875,6 +2875,50 @@ void PGMap::get_health_checks(
       d.detail.swap(detail);
     }
   }
+
+  // PG_SLOW_SNAP_TRIMMING
+  if (!pg_stat.empty() && cct->_conf->mon_osd_snap_trim_queue_warn_on > 0) {
+    uint32_t snapthreshold = cct->_conf->mon_osd_snap_trim_queue_warn_on;
+    uint64_t snaptrimq_exceeded = 0;
+    uint32_t longest_queue = 0;
+    const pg_t* longest_q_pg = nullptr;
+    list<string> detail;
+
+    for (auto& i: pg_stat) {
+      uint32_t current_len = i.second.snaptrimq_len;
+      if (current_len >= snapthreshold) {
+        snaptrimq_exceeded++;
+        if (longest_queue <= current_len) {
+          longest_q_pg = &i.first;
+          longest_queue = current_len;
+        }
+        if (detail.size() < max - 1) {
+          stringstream ss;
+          ss << "snap trim queue for pg " << i.first << " at " << current_len;
+          detail.push_back(ss.str());
+          continue;
+        }
+        if (detail.size() < max) {
+          detail.push_back("...more pgs affected");
+          continue;
+        }
+      }
+    }
+
+    if (snaptrimq_exceeded) {
+      {
+         ostringstream ss;
+         ss << "longest queue on pg " << *longest_q_pg << " at " << longest_queue;
+         detail.push_back(ss.str());
+      }
+
+      stringstream ss;
+      ss << "snap trim queue for " << snaptrimq_exceeded << " pg(s) >= " << snapthreshold << " (mon_osd_snap_trim_queue_warn_on)";
+      auto& d = checks->add("PG_SLOW_SNAP_TRIMMING", HEALTH_WARN, ss.str());
+      detail.push_back("try decreasing \"osd snap trim sleep\" and/or increasing \"osd pg max concurrent snap trims\".");
+      d.detail.swap(detail);
+    }
+  }
 }
 
 int process_pg_map_command(