mon/OSDMonitor: prune purged snaps

Be a bit careful here because the mon has to do some bookkeeping to avoid
pruning things twice.  If the PGMapDigest set appears obviously stale,
skip some work (looking at this particular interval) until it is not
obviously stale--move onto the next interval instead.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index 30de345e684fe64d1d4fb3dd3f8ecd834b9f778c..72d0fbb44fbb05279c2f341ba128a1d08e66c2f3 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1153,6 +1153,10 @@ std::vector<Option> get_global_options() {
     .set_default(true)
     .set_description("Enable POOL_APP_NOT_ENABLED health check"),
 
+    Option("mon_max_snap_prune_per_epoch", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(100)
+    .set_description("Max number of pruned snaps we will process in a single OSDMap epoch"),
+
     Option("mon_min_osdmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(500)
     .set_description(""),

diff --git a/src/mon/MgrStatMonitor.h b/src/mon/MgrStatMonitor.h
index a4c4e589180c1848d32a5f90e4c5252b6b6498f5..6442965c7d55cf61b3f6d42f4366cb6a5d9f6a70 100644 (file)
--- a/src/mon/MgrStatMonitor.h
+++ b/src/mon/MgrStatMonitor.h
@@ -73,6 +73,10 @@ public:
     return nullptr;
   }
 
+  const PGMapDigest& get_digest() {
+    return digest;
+  }
+
   ceph_statfs get_statfs(OSDMap& osdmap,
                         boost::optional<int64_t> data_pool) const {
     return digest.get_statfs(osdmap, data_pool);

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index c441a02f6123bce7e1dfcdbd90327b2dd0adb45d..4ec000eac50f346123e478cfe0f3b30512c377fa 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -1311,6 +1311,17 @@ void OSDMonitor::encode_pending(MonitorDBStore::TransactionRef t)
        t->put(OSD_SNAP_PREFIX, k, v);
       }
     }
+    for (auto& i : pending_inc.new_purged_snaps) {
+      for (auto q = i.second.begin();
+          q != i.second.end();
+          ++q) {
+       bufferlist v;
+       string k = make_snap_purged_key_value(i.first, q.get_start(),
+                                             q.get_len(), pending_inc.epoch,
+                                             &v);
+       t->put(OSD_SNAP_PREFIX, k, v);
+      }
+    }
   }
 
   // health
@@ -3471,6 +3482,10 @@ void OSDMonitor::tick()
     }
   }
 
+  if (try_prune_purged_snaps()) {
+    do_propose = true;
+  }
+
   if (update_pools_status())
     do_propose = true;
 
@@ -4957,6 +4972,7 @@ string OSDMonitor::make_snap_key(int64_t pool, snapid_t snap)
   return k;
 }
 
+
 string OSDMonitor::make_snap_key_value(
   int64_t pool, snapid_t snap, snapid_t num,
   epoch_t epoch, bufferlist *v)
@@ -4969,6 +4985,130 @@ string OSDMonitor::make_snap_key_value(
   return make_snap_key(pool, snap + num - 1);
 }
 
+string OSDMonitor::make_snap_purged_key(int64_t pool, snapid_t snap)
+{
+  char k[80];
+  snprintf(k, sizeof(k), "purged_snap_%lld_%016llx",
+          (unsigned long long)pool, (unsigned long long)snap);
+  return k;
+}
+string OSDMonitor::make_snap_purged_key_value(
+  int64_t pool, snapid_t snap, snapid_t num,
+  epoch_t epoch, bufferlist *v)
+{
+  // encode the *last* epoch in the key so that we can use forward
+  // iteration only to search for an epoch in an interval.
+  ::encode(snap, *v);
+  ::encode(snap + num, *v);
+  ::encode(epoch, *v);
+  return make_snap_purged_key(pool, snap + num - 1);
+}
+
+int OSDMonitor::lookup_pruned_snap(int64_t pool, snapid_t snap,
+                                  snapid_t *begin, snapid_t *end)
+{
+  string k = make_snap_key(pool, snap);
+  auto it = mon->store->get_iterator(OSD_SNAP_PREFIX);
+  it->lower_bound(k);
+  if (!it->valid()) {
+    return -ENOENT;
+  }
+  if (it->key().find(OSD_SNAP_PREFIX) != 0) {
+    return -ENOENT;
+  }
+  bufferlist v = it->value();
+  auto p = v.begin();
+  ::decode(*begin, p);
+  ::decode(*end, p);
+  if (snap < *begin || snap >= *end) {
+    return -ENOENT;
+  }
+  return 0;
+}
+
+bool OSDMonitor::try_prune_purged_snaps()
+{
+  if (!mon->mgrstatmon()->is_readable()) {
+    return false;
+  }
+  if (osdmap.require_osd_release < CEPH_RELEASE_MIMIC) {
+    return false;
+  }
+  if (!pending_inc.new_purged_snaps.empty()) {
+    return false;  // we already pruned for this epoch
+  }
+
+  unsigned max_prune = cct->_conf->get_val<uint64_t>(
+    "mon_max_snap_prune_per_epoch");
+  if (!max_prune) {
+    max_prune = 100000;
+  }
+  dout(10) << __func__ << " max_prune " << max_prune << dendl;
+
+  unsigned actually_pruned = 0;
+  auto& purged_snaps = mon->mgrstatmon()->get_digest().purged_snaps;
+  for (auto& p : osdmap.get_pools()) {
+    auto q = purged_snaps.find(p.first);
+    if (q == purged_snaps.end()) {
+      continue;
+    }
+    auto& purged = q->second;
+    if (purged.empty()) {
+      dout(20) << __func__ << " " << p.first << " nothing purged" << dendl;
+      continue;
+    }
+    dout(20) << __func__ << " pool " << p.first << " purged " << purged << dendl;
+    OSDMap::snap_interval_set_t to_prune;
+    unsigned maybe_pruned = actually_pruned;
+    for (auto i = purged.begin(); i != purged.end(); ++i) {
+      snapid_t begin = i.get_start();
+      auto end = i.get_start() + i.get_len();
+      snapid_t pbegin = 0, pend = 0;
+      int r = lookup_pruned_snap(p.first, begin, &pbegin, &pend);
+      if (r == 0) {
+       // already purged.
+       // be a bit aggressive about backing off here, because the mon may
+       // do a lot of work going through this set, and if we know the
+       // purged set from the OSDs is at least *partly* stale we may as
+       // well wait for it to be fresh.
+       dout(20) << __func__ << "  we've already pruned " << pbegin
+                << "~" << (pend - pbegin) << dendl;
+       break;  // next pool
+      }
+      if (pbegin && pbegin < end) {
+       // the tail of [begin,end) is purged; shorten the range
+       assert(pbegin > begin);
+       end = pbegin;
+      }
+      to_prune.insert(begin, end - begin);
+      maybe_pruned += end - begin;
+      if (maybe_pruned >= max_prune) {
+       break;
+      }
+    }
+    if (!to_prune.empty()) {
+      // PGs may still be reporting things as purged that we have already
+      // pruned from removed_snaps_queue.
+      OSDMap::snap_interval_set_t actual;
+      auto r = osdmap.removed_snaps_queue.find(p.first);
+      if (r != osdmap.removed_snaps_queue.end()) {
+       actual.intersection_of(to_prune, r->second);
+      }
+      actually_pruned += actual.size();
+      dout(10) << __func__ << " pool " << p.first << " reports pruned " << to_prune
+              << ", actual pruned " << actual << dendl;
+      if (!actual.empty()) {
+       pending_inc.new_purged_snaps[p.first].swap(actual);
+      }
+    }
+    if (actually_pruned >= max_prune) {
+      break;
+    }
+  }
+  dout(10) << __func__ << " actually pruned " << actually_pruned << dendl;
+  return !!actually_pruned;
+}
+
 bool OSDMonitor::update_pools_status()
 {
   if (!mon->mgrstatmon()->is_readable())

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index abb1926b167a4ea9ef5d6f35eba425ff87489bb7..24052d725534cbb02555eae6d5d978bb3a3d3e98 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -363,6 +363,12 @@ private:
   string make_snap_key(int64_t pool, snapid_t snap);
   string make_snap_key_value(int64_t pool, snapid_t snap, snapid_t num,
                             epoch_t epoch, bufferlist *v);
+  string make_snap_purged_key(int64_t pool, snapid_t snap);
+  string make_snap_purged_key_value(int64_t pool, snapid_t snap, snapid_t num,
+                                   epoch_t epoch, bufferlist *v);
+  bool try_prune_purged_snaps();
+  int lookup_pruned_snap(int64_t pool, snapid_t snap,
+                        snapid_t *begin, snapid_t *end);
 
   bool prepare_set_flag(MonOpRequestRef op, int flag);
   bool prepare_unset_flag(MonOpRequestRef op, int flag);