From 87b539c2b65c80a105e1c374ff88cb69b571686f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 10 Jun 2019 10:12:04 -0500 Subject: [PATCH] osd: record purged_snaps when we store new maps When we get a new map, record the (new) purged_snaps. Only do this if the OSD has purged_snaps that are in sync with the latest OSDMap. That means that after an upgrade, if the OSD didn't sync the old purged_snaps on startup, it won't sync anything until it *next* starts up. Signed-off-by: Sage Weil --- src/osd/OSD.cc | 17 ++++++ src/osd/SnapMapper.cc | 120 ++++++++++++++++++++++++++++++++++++++++++ src/osd/SnapMapper.h | 24 +++++++++ 3 files changed, 161 insertions(+) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 7c1aaacab179c..e316c1fc2f46b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -7498,6 +7498,8 @@ void OSD::handle_osd_map(MOSDMap *m) ObjectStore::Transaction t; uint64_t txn_size = 0; + map> purged_snaps; + // store new maps: queue for disk and put in the osdmap cache epoch_t start = std::max(superblock.newest_map + 1, first); for (epoch_t e = start; e <= last; e++) { @@ -7515,6 +7517,8 @@ void OSD::handle_osd_map(MOSDMap *m) o->decode(bl); + purged_snaps[e] = o->get_new_purged_snaps(); + ghobject_t fulloid = get_osdmap_pobject_name(e); t.write(coll_t::meta(), fulloid, 0, bl.length(), bl); added_maps[e] = add_map(o); @@ -7574,6 +7578,7 @@ void OSD::handle_osd_map(MOSDMap *m) break; } got_full_map(e); + purged_snaps[e] = o->get_new_purged_snaps(); ghobject_t fulloid = get_osdmap_pobject_name(e); t.write(coll_t::meta(), fulloid, 0, fbl.length(), fbl); @@ -7668,6 +7673,18 @@ void OSD::handle_osd_map(MOSDMap *m) dout(20) << __func__ << " pg_num_history " << pg_num_history << dendl; } + // record new purged_snaps + if (superblock.purged_snaps_last == start - 1) { + SnapMapper::record_purged_snaps(cct, store, service.meta_ch, + make_snapmapper_oid(), &t, + purged_snaps); + superblock.purged_snaps_last = last; + } else { + dout(10) << __func__ << " superblock purged_snaps_last is " + << superblock.purged_snaps_last + << ", not recording new purged_snaps" << dendl; + } + // superblock and commit write_superblock(t); t.register_on_commit(new C_OnMapCommit(this, start, last, m)); diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc index 968caff370b2e..c69da102d4021 100644 --- a/src/osd/SnapMapper.cc +++ b/src/osd/SnapMapper.cc @@ -25,6 +25,9 @@ const string SnapMapper::LEGACY_MAPPING_PREFIX = "MAP_"; const string SnapMapper::MAPPING_PREFIX = "SNA_"; const string SnapMapper::OBJECT_PREFIX = "OBJ_"; +const char *SnapMapper::PURGED_SNAP_EPOCH_PREFIX = "PSE_"; +const char *SnapMapper::PURGED_SNAP_PREFIX = "PSN_"; + /* We have a bidirectional mapping, (1) from each snap+obj to object, @@ -400,6 +403,123 @@ int SnapMapper::get_snaps( } +// -- purged snaps -- + +string SnapMapper::make_purged_snap_key(int64_t pool, snapid_t last) +{ + char k[80]; + snprintf(k, sizeof(k), "%s_%llu_%016llx", PURGED_SNAP_PREFIX, + (unsigned long long)pool, (unsigned long long)last); + return k; +} + +void SnapMapper::make_purged_snap_key_value( + int64_t pool, snapid_t begin, snapid_t end, map *m) +{ + string k = make_purged_snap_key(pool, end - 1); + auto& v = (*m)[k]; + ceph::encode(begin, v); + ceph::encode(end, v); +} + +int SnapMapper::_lookup_purged_snap( + CephContext *cct, + ObjectStore *store, + ObjectStore::CollectionHandle& ch, + const ghobject_t& hoid, + int64_t pool, snapid_t snap, + snapid_t *begin, snapid_t *end) +{ + string k = make_purged_snap_key(pool, snap); + auto it = store->get_omap_iterator(ch, hoid); + it->lower_bound(k); + if (!it->valid()) { + dout(20) << __func__ << " pool " << pool << " snap " << snap + << " key '" << k << "' lower_bound not found" << dendl; + return -ENOENT; + } + if (it->key().find(PURGED_SNAP_PREFIX) != 0) { + dout(20) << __func__ << " pool " << pool << " snap " << snap + << " key '" << k << "' lower_bound got mismatched prefix '" + << it->key() << "'" << dendl; + return -ENOENT; + } + bufferlist v = it->value(); + auto p = v.cbegin(); + decode(*begin, p); + decode(*end, p); + if (snap < *begin || snap >= *end) { + dout(20) << __func__ << " pool " << pool << " snap " << snap + << " found [" << *begin << "," << *end << "), no overlap" << dendl; + return -ENOENT; + } + return 0; +} + +void SnapMapper::record_purged_snaps( + CephContext *cct, + ObjectStore *store, + ObjectStore::CollectionHandle& ch, + ghobject_t hoid, + ObjectStore::Transaction *t, + map> purged_snaps) +{ + dout(10) << __func__ << " purged_snaps " << purged_snaps << dendl; + map m; + set rm; + for (auto& [epoch, bypool] : purged_snaps) { + // store per-epoch key + char ek[80]; + snprintf(ek, sizeof(ek), "%s_%08lx", PURGED_SNAP_EPOCH_PREFIX, + (unsigned long)epoch); + ceph::encode(bypool, m[ek]); + + // index by (pool, snap) + for (auto& [pool, snaps] : bypool) { + for (auto i = snaps.begin(); + i != snaps.end(); + ++i) { + snapid_t begin = i.get_start(); + snapid_t end = i.get_end(); + snapid_t before_begin, before_end; + snapid_t after_begin, after_end; + int b = _lookup_purged_snap(cct, store, ch, hoid, + pool, begin - 1, &before_begin, &before_end); + int a = _lookup_purged_snap(cct, store, ch, hoid, + pool, end, &after_begin, &after_end); + if (!b && !a) { + dout(10) << __func__ + << " [" << begin << "," << end << ") - joins [" + << before_begin << "," << before_end << ") and [" + << after_begin << "," << after_end << ")" << dendl; + // erase only the begin record; we'll overwrite the end one + rm.insert(make_purged_snap_key(pool, before_end - 1)); + make_purged_snap_key_value(pool, before_begin, after_end, &m); + } else if (!b) { + dout(10) << __func__ + << " [" << begin << "," << end << ") - join with earlier [" + << before_begin << "," << before_end << ")" << dendl; + rm.insert(make_purged_snap_key(pool, before_end - 1)); + make_purged_snap_key_value(pool, before_begin, end, &m); + } else if (!a) { + dout(10) << __func__ + << " [" << begin << "," << end << ") - join with later [" + << after_begin << "," << after_end << ")" << dendl; + // overwrite after record + make_purged_snap_key_value(pool, begin, after_end, &m); + } else { + make_purged_snap_key_value(pool, begin, end, &m); + } + } + } + } + t->omap_rmkeys(ch->cid, hoid, rm); + t->omap_setkeys(ch->cid, hoid, m); + dout(10) << __func__ << " rm " << rm.size() << " keys, set " << m.size() + << " keys" << dendl; +} + + // ------------------------------------- // legacy conversion/support diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h index d8899b12f3469..da84462add9ee 100644 --- a/src/osd/SnapMapper.h +++ b/src/osd/SnapMapper.h @@ -26,6 +26,7 @@ #include "include/encoding.h" #include "include/object.h" #include "os/ObjectStore.h" +#include "osd/OSDMap.h" class OSDriver : public MapCacher::StoreDriver { ObjectStore *os; @@ -134,6 +135,8 @@ public: static const std::string LEGACY_MAPPING_PREFIX; static const std::string MAPPING_PREFIX; static const std::string OBJECT_PREFIX; + static const char *PURGED_SNAP_EPOCH_PREFIX; + static const char *PURGED_SNAP_PREFIX; static int convert_legacy( CephContext *cct, @@ -142,7 +145,28 @@ public: ghobject_t hoid, unsigned max); + static void record_purged_snaps( + CephContext *cct, + ObjectStore *store, + ObjectStore::CollectionHandle& ch, + ghobject_t hoid, + ObjectStore::Transaction *t, + map> purged_snaps); + private: + static int _lookup_purged_snap( + CephContext *cct, + ObjectStore *store, + ObjectStore::CollectionHandle& ch, + const ghobject_t& hoid, + int64_t pool, snapid_t snap, + snapid_t *begin, snapid_t *end); + static void make_purged_snap_key_value( + int64_t pool, snapid_t begin, + snapid_t end, map *m); + static string make_purged_snap_key(int64_t pool, snapid_t last); + + MapCacher::MapCacher backend; static std::string get_legacy_prefix(snapid_t snap); -- 2.39.5