From: Sage Weil <sage@redhat.com>
Date: Sat, 28 Oct 2017 19:58:44 +0000 (-0500)
Subject: mon/OSDMonitor: prune purged snaps
X-Git-Tag: v13.0.2~844^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9607a2db466835406d6fb73ad6776eccdf3a7d6c;p=ceph.git

mon/OSDMonitor: prune purged snaps

Be a bit careful here because the mon has to do some bookkeeping to avoid
pruning things twice.  If the PGMapDigest set appears obviously stale,
skip some work (looking at this particular interval) until it is not
obviously stale--move onto the next interval instead.

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/src/common/options.cc b/src/common/options.cc
index 30de345e684f..72d0fbb44fbb 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1153,6 +1153,10 @@ std::vector<Option> get_global_options() {
     .set_default(true)
     .set_description("Enable POOL_APP_NOT_ENABLED health check"),
 
+    Option("mon_max_snap_prune_per_epoch", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(100)
+    .set_description("Max number of pruned snaps we will process in a single OSDMap epoch"),
+
     Option("mon_min_osdmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(500)
     .set_description(""),
diff --git a/src/mon/MgrStatMonitor.h b/src/mon/MgrStatMonitor.h
index a4c4e589180c..6442965c7d55 100644
--- a/src/mon/MgrStatMonitor.h
+++ b/src/mon/MgrStatMonitor.h
@@ -73,6 +73,10 @@ public:
     return nullptr;
   }
 
+  const PGMapDigest& get_digest() {
+    return digest;
+  }
+
   ceph_statfs get_statfs(OSDMap& osdmap,
 			 boost::optional<int64_t> data_pool) const {
     return digest.get_statfs(osdmap, data_pool);
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index c441a02f6123..4ec000eac50f 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -1311,6 +1311,17 @@ void OSDMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 	t->put(OSD_SNAP_PREFIX, k, v);
       }
     }
+    for (auto& i : pending_inc.new_purged_snaps) {
+      for (auto q = i.second.begin();
+	   q != i.second.end();
+	   ++q) {
+	bufferlist v;
+	string k = make_snap_purged_key_value(i.first, q.get_start(),
+					      q.get_len(), pending_inc.epoch,
+					      &v);
+	t->put(OSD_SNAP_PREFIX, k, v);
+      }
+    }
   }
 
   // health
@@ -3471,6 +3482,10 @@ void OSDMonitor::tick()
     }
   }
 
+  if (try_prune_purged_snaps()) {
+    do_propose = true;
+  }
+
   if (update_pools_status())
     do_propose = true;
 
@@ -4957,6 +4972,7 @@ string OSDMonitor::make_snap_key(int64_t pool, snapid_t snap)
   return k;
 }
 
+
 string OSDMonitor::make_snap_key_value(
   int64_t pool, snapid_t snap, snapid_t num,
   epoch_t epoch, bufferlist *v)
@@ -4969,6 +4985,130 @@ string OSDMonitor::make_snap_key_value(
   return make_snap_key(pool, snap + num - 1);
 }
 
+string OSDMonitor::make_snap_purged_key(int64_t pool, snapid_t snap)
+{
+  char k[80];
+  snprintf(k, sizeof(k), "purged_snap_%lld_%016llx",
+	   (unsigned long long)pool, (unsigned long long)snap);
+  return k;
+}
+string OSDMonitor::make_snap_purged_key_value(
+  int64_t pool, snapid_t snap, snapid_t num,
+  epoch_t epoch, bufferlist *v)
+{
+  // encode the *last* epoch in the key so that we can use forward
+  // iteration only to search for an epoch in an interval.
+  ::encode(snap, *v);
+  ::encode(snap + num, *v);
+  ::encode(epoch, *v);
+  return make_snap_purged_key(pool, snap + num - 1);
+}
+
+int OSDMonitor::lookup_pruned_snap(int64_t pool, snapid_t snap,
+				   snapid_t *begin, snapid_t *end)
+{
+  string k = make_snap_key(pool, snap);
+  auto it = mon->store->get_iterator(OSD_SNAP_PREFIX);
+  it->lower_bound(k);
+  if (!it->valid()) {
+    return -ENOENT;
+  }
+  if (it->key().find(OSD_SNAP_PREFIX) != 0) {
+    return -ENOENT;
+  }
+  bufferlist v = it->value();
+  auto p = v.begin();
+  ::decode(*begin, p);
+  ::decode(*end, p);
+  if (snap < *begin || snap >= *end) {
+    return -ENOENT;
+  }
+  return 0;
+}
+
+bool OSDMonitor::try_prune_purged_snaps()
+{
+  if (!mon->mgrstatmon()->is_readable()) {
+    return false;
+  }
+  if (osdmap.require_osd_release < CEPH_RELEASE_MIMIC) {
+    return false;
+  }
+  if (!pending_inc.new_purged_snaps.empty()) {
+    return false;  // we already pruned for this epoch
+  }
+
+  unsigned max_prune = cct->_conf->get_val<uint64_t>(
+    "mon_max_snap_prune_per_epoch");
+  if (!max_prune) {
+    max_prune = 100000;
+  }
+  dout(10) << __func__ << " max_prune " << max_prune << dendl;
+
+  unsigned actually_pruned = 0;
+  auto& purged_snaps = mon->mgrstatmon()->get_digest().purged_snaps;
+  for (auto& p : osdmap.get_pools()) {
+    auto q = purged_snaps.find(p.first);
+    if (q == purged_snaps.end()) {
+      continue;
+    }
+    auto& purged = q->second;
+    if (purged.empty()) {
+      dout(20) << __func__ << " " << p.first << " nothing purged" << dendl;
+      continue;
+    }
+    dout(20) << __func__ << " pool " << p.first << " purged " << purged << dendl;
+    OSDMap::snap_interval_set_t to_prune;
+    unsigned maybe_pruned = actually_pruned;
+    for (auto i = purged.begin(); i != purged.end(); ++i) {
+      snapid_t begin = i.get_start();
+      auto end = i.get_start() + i.get_len();
+      snapid_t pbegin = 0, pend = 0;
+      int r = lookup_pruned_snap(p.first, begin, &pbegin, &pend);
+      if (r == 0) {
+	// already purged.
+	// be a bit aggressive about backing off here, because the mon may
+	// do a lot of work going through this set, and if we know the
+	// purged set from the OSDs is at least *partly* stale we may as
+	// well wait for it to be fresh.
+	dout(20) << __func__ << "  we've already pruned " << pbegin
+		 << "~" << (pend - pbegin) << dendl;
+	break;  // next pool
+      }
+      if (pbegin && pbegin < end) {
+	// the tail of [begin,end) is purged; shorten the range
+	assert(pbegin > begin);
+	end = pbegin;
+      }
+      to_prune.insert(begin, end - begin);
+      maybe_pruned += end - begin;
+      if (maybe_pruned >= max_prune) {
+	break;
+      }
+    }
+    if (!to_prune.empty()) {
+      // PGs may still be reporting things as purged that we have already
+      // pruned from removed_snaps_queue.
+      OSDMap::snap_interval_set_t actual;
+      auto r = osdmap.removed_snaps_queue.find(p.first);
+      if (r != osdmap.removed_snaps_queue.end()) {
+	actual.intersection_of(to_prune, r->second);
+      }
+      actually_pruned += actual.size();
+      dout(10) << __func__ << " pool " << p.first << " reports pruned " << to_prune
+	       << ", actual pruned " << actual << dendl;
+      if (!actual.empty()) {
+	pending_inc.new_purged_snaps[p.first].swap(actual);
+      }
+    }
+    if (actually_pruned >= max_prune) {
+      break;
+    }
+  }
+  dout(10) << __func__ << " actually pruned " << actually_pruned << dendl;
+  return !!actually_pruned;
+}
+
 bool OSDMonitor::update_pools_status()
 {
   if (!mon->mgrstatmon()->is_readable())
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index abb1926b167a..24052d725534 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -363,6 +363,12 @@ private:
   string make_snap_key(int64_t pool, snapid_t snap);
   string make_snap_key_value(int64_t pool, snapid_t snap, snapid_t num,
 			     epoch_t epoch, bufferlist *v);
+  string make_snap_purged_key(int64_t pool, snapid_t snap);
+  string make_snap_purged_key_value(int64_t pool, snapid_t snap, snapid_t num,
+				    epoch_t epoch, bufferlist *v);
+  bool try_prune_purged_snaps();
+  int lookup_pruned_snap(int64_t pool, snapid_t snap,
+			 snapid_t *begin, snapid_t *end);
 
   bool prepare_set_flag(MonOpRequestRef op, int flag);
   bool prepare_unset_flag(MonOpRequestRef op, int flag);