osd: record purged_snaps when we store new maps

When we get a new map, record the (new) purged_snaps.

Only do this if the OSD has purged_snaps that are in sync with the latest
OSDMap.  That means that after an upgrade, if the OSD didn't sync the
old purged_snaps on startup, it won't sync anything until it *next* starts
up.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 7c1aaacab179cc01e62cfef9a956f77c8b56a406..e316c1fc2f46b8b4e37f2dc77bf433f917cf75fb 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7498,6 +7498,8 @@ void OSD::handle_osd_map(MOSDMap *m)
   ObjectStore::Transaction t;
   uint64_t txn_size = 0;
 
+  map<epoch_t,mempool::osdmap::map<int64_t,snap_interval_set_t>> purged_snaps;
+
   // store new maps: queue for disk and put in the osdmap cache
   epoch_t start = std::max(superblock.newest_map + 1, first);
   for (epoch_t e = start; e <= last; e++) {
@@ -7515,6 +7517,8 @@ void OSD::handle_osd_map(MOSDMap *m)
 
       o->decode(bl);
 
+      purged_snaps[e] = o->get_new_purged_snaps();
+
       ghobject_t fulloid = get_osdmap_pobject_name(e);
       t.write(coll_t::meta(), fulloid, 0, bl.length(), bl);
       added_maps[e] = add_map(o);
@@ -7574,6 +7578,7 @@ void OSD::handle_osd_map(MOSDMap *m)
        break;
       }
       got_full_map(e);
+      purged_snaps[e] = o->get_new_purged_snaps();
 
       ghobject_t fulloid = get_osdmap_pobject_name(e);
       t.write(coll_t::meta(), fulloid, 0, fbl.length(), fbl);
@@ -7668,6 +7673,18 @@ void OSD::handle_osd_map(MOSDMap *m)
     dout(20) << __func__ << " pg_num_history " << pg_num_history << dendl;
   }
 
+  // record new purged_snaps
+  if (superblock.purged_snaps_last == start - 1) {
+    SnapMapper::record_purged_snaps(cct, store, service.meta_ch,
+                                   make_snapmapper_oid(), &t,
+                                   purged_snaps);
+    superblock.purged_snaps_last = last;
+  } else {
+    dout(10) << __func__ << " superblock purged_snaps_last is "
+            << superblock.purged_snaps_last
+            << ", not recording new purged_snaps" << dendl;
+  }
+
   // superblock and commit
   write_superblock(t);
   t.register_on_commit(new C_OnMapCommit(this, start, last, m));

diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index 968caff370b2e77eb363a723f22952d557acf712..c69da102d40216654e2b71be548bf9d176aab489 100644 (file)
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -25,6 +25,9 @@ const string SnapMapper::LEGACY_MAPPING_PREFIX = "MAP_";
 const string SnapMapper::MAPPING_PREFIX = "SNA_";
 const string SnapMapper::OBJECT_PREFIX = "OBJ_";
 
+const char *SnapMapper::PURGED_SNAP_EPOCH_PREFIX = "PSE_";
+const char *SnapMapper::PURGED_SNAP_PREFIX = "PSN_";
+
 /*
 
   We have a bidirectional mapping, (1) from each snap+obj to object,
@@ -400,6 +403,123 @@ int SnapMapper::get_snaps(
 }
 
 
+// -- purged snaps --
+
+string SnapMapper::make_purged_snap_key(int64_t pool, snapid_t last)
+{
+  char k[80];
+  snprintf(k, sizeof(k), "%s_%llu_%016llx", PURGED_SNAP_PREFIX,
+          (unsigned long long)pool, (unsigned long long)last);
+  return k;
+}
+
+void SnapMapper::make_purged_snap_key_value(
+  int64_t pool, snapid_t begin, snapid_t end, map<string,bufferlist> *m)
+{
+  string k = make_purged_snap_key(pool, end - 1);
+  auto& v = (*m)[k];
+  ceph::encode(begin, v);
+  ceph::encode(end, v);
+}
+
+int SnapMapper::_lookup_purged_snap(
+  CephContext *cct,
+  ObjectStore *store,
+  ObjectStore::CollectionHandle& ch,
+  const ghobject_t& hoid,
+  int64_t pool, snapid_t snap,
+  snapid_t *begin, snapid_t *end)
+{
+  string k = make_purged_snap_key(pool, snap);
+  auto it = store->get_omap_iterator(ch, hoid);
+  it->lower_bound(k);
+  if (!it->valid()) {
+    dout(20) << __func__ << " pool " << pool << " snap " << snap
+            << " key '" << k << "' lower_bound not found" << dendl;
+    return -ENOENT;
+  }
+  if (it->key().find(PURGED_SNAP_PREFIX) != 0) {
+    dout(20) << __func__ << " pool " << pool << " snap " << snap
+            << " key '" << k << "' lower_bound got mismatched prefix '"
+            << it->key() << "'" << dendl;
+    return -ENOENT;
+  }
+  bufferlist v = it->value();
+  auto p = v.cbegin();
+  decode(*begin, p);
+  decode(*end, p);
+  if (snap < *begin || snap >= *end) {
+    dout(20) << __func__ << " pool " << pool << " snap " << snap
+            << " found [" << *begin << "," << *end << "), no overlap" << dendl;
+    return -ENOENT;
+  }
+  return 0;
+}
+
+void SnapMapper::record_purged_snaps(
+  CephContext *cct,
+  ObjectStore *store,
+  ObjectStore::CollectionHandle& ch,
+  ghobject_t hoid,
+  ObjectStore::Transaction *t,
+  map<epoch_t,mempool::osdmap::map<int64_t,snap_interval_set_t>> purged_snaps)
+{
+  dout(10) << __func__ << " purged_snaps " << purged_snaps << dendl;
+  map<string,bufferlist> m;
+  set<string> rm;
+  for (auto& [epoch, bypool] : purged_snaps) {
+    // store per-epoch key
+    char ek[80];
+    snprintf(ek, sizeof(ek), "%s_%08lx", PURGED_SNAP_EPOCH_PREFIX,
+            (unsigned long)epoch);
+    ceph::encode(bypool, m[ek]);
+
+    // index by (pool, snap)
+    for (auto& [pool, snaps] : bypool) {
+      for (auto i = snaps.begin();
+          i != snaps.end();
+          ++i) {
+       snapid_t begin = i.get_start();
+       snapid_t end = i.get_end();
+       snapid_t before_begin, before_end;
+       snapid_t after_begin, after_end;
+       int b = _lookup_purged_snap(cct, store, ch, hoid,
+                                   pool, begin - 1, &before_begin, &before_end);
+       int a = _lookup_purged_snap(cct, store, ch, hoid,
+                                   pool, end, &after_begin, &after_end);
+       if (!b && !a) {
+         dout(10) << __func__
+                  << " [" << begin << "," << end << ") - joins ["
+                  << before_begin << "," << before_end << ") and ["
+                  << after_begin << "," << after_end << ")" << dendl;
+         // erase only the begin record; we'll overwrite the end one
+         rm.insert(make_purged_snap_key(pool, before_end - 1));
+         make_purged_snap_key_value(pool, before_begin, after_end, &m);
+       } else if (!b) {
+         dout(10) << __func__
+                  << " [" << begin << "," << end << ") - join with earlier ["
+                  << before_begin << "," << before_end << ")" << dendl;
+         rm.insert(make_purged_snap_key(pool, before_end - 1));
+         make_purged_snap_key_value(pool, before_begin, end, &m);
+       } else if (!a) {
+         dout(10) << __func__
+                  << " [" << begin << "," << end << ") - join with later ["
+                  << after_begin << "," << after_end << ")" << dendl;
+         // overwrite after record
+         make_purged_snap_key_value(pool, begin, after_end, &m);
+       } else {
+         make_purged_snap_key_value(pool, begin, end, &m);
+       }
+      }
+    }
+  }
+  t->omap_rmkeys(ch->cid, hoid, rm);
+  t->omap_setkeys(ch->cid, hoid, m);
+  dout(10) << __func__ << " rm " << rm.size() << " keys, set " << m.size()
+          << " keys" << dendl;
+}
+
+
 // -------------------------------------
 // legacy conversion/support
 

diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h
index d8899b12f3469b553b4b33920f19727e2948a5a2..da84462add9eefeeb4125766709114d2cd1835a7 100644 (file)
--- a/src/osd/SnapMapper.h
+++ b/src/osd/SnapMapper.h
@@ -26,6 +26,7 @@
 #include "include/encoding.h"
 #include "include/object.h"
 #include "os/ObjectStore.h"
+#include "osd/OSDMap.h"
 
 class OSDriver : public MapCacher::StoreDriver<std::string, bufferlist> {
   ObjectStore *os;
@@ -134,6 +135,8 @@ public:
   static const std::string LEGACY_MAPPING_PREFIX;
   static const std::string MAPPING_PREFIX;
   static const std::string OBJECT_PREFIX;
+  static const char *PURGED_SNAP_EPOCH_PREFIX;
+  static const char *PURGED_SNAP_PREFIX;
 
   static int convert_legacy(
     CephContext *cct,
@@ -142,7 +145,28 @@ public:
     ghobject_t hoid,
     unsigned max);
 
+  static void record_purged_snaps(
+    CephContext *cct,
+    ObjectStore *store,
+    ObjectStore::CollectionHandle& ch,
+    ghobject_t hoid,
+    ObjectStore::Transaction *t,
+    map<epoch_t,mempool::osdmap::map<int64_t,snap_interval_set_t>> purged_snaps);
+
 private:
+  static int _lookup_purged_snap(
+    CephContext *cct,
+    ObjectStore *store,
+    ObjectStore::CollectionHandle& ch,
+    const ghobject_t& hoid,
+    int64_t pool, snapid_t snap,
+    snapid_t *begin, snapid_t *end);
+  static void make_purged_snap_key_value(
+    int64_t pool, snapid_t begin,
+    snapid_t end, map<string,bufferlist> *m);
+  static string make_purged_snap_key(int64_t pool, snapid_t last);
+
+
   MapCacher::MapCacher<std::string, bufferlist> backend;
 
   static std::string get_legacy_prefix(snapid_t snap);