From 1c3fe5649d425e7d6d52fe7f9551606082add561 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 5 Jun 2009 10:43:06 -0700 Subject: [PATCH] osd: purge snaps on a per-pool basis Maintain a refcounted PGPool object in the OSD to track pool-wide state (such as the latest SnapContext, snap trimming info). Doesn't trim per-pool snaps just yet. --- src/TODO | 3 ++ src/mds/SnapServer.cc | 53 +++++++++++++++-------- src/mds/SnapServer.h | 2 +- src/messages/MRemoveSnaps.h | 4 +- src/mon/OSDMonitor.cc | 58 +++++++++++++++---------- src/osd/OSD.cc | 85 ++++++++++++++++++++++++++++++------- src/osd/OSD.h | 7 ++- src/osd/OSDMap.cc | 9 ++-- src/osd/OSDMap.h | 32 ++------------ src/osd/PG.cc | 2 +- src/osd/PG.h | 41 ++++++++++++++---- src/osd/ReplicatedPG.cc | 8 ++-- src/osd/ReplicatedPG.h | 4 +- src/osd/osd_types.h | 5 ++- 14 files changed, 204 insertions(+), 109 deletions(-) diff --git a/src/TODO b/src/TODO index 68ed552a510eb..ea76d3f4ffbd4 100644 --- a/src/TODO +++ b/src/TODO @@ -34,6 +34,9 @@ later - rename over old files should flush data, or revert back to old contents rados +- snaps + - use default pool contexts + - make snap removal work with the default pools (fix new_removed_snaps logic in OSD.cc) - merge pgs - destroy pg_pools - autosize pg_pools? diff --git a/src/mds/SnapServer.cc b/src/mds/SnapServer.cc index a7782dbb08cbe..e4d06cc9d50c6 100644 --- a/src/mds/SnapServer.cc +++ b/src/mds/SnapServer.cc @@ -114,7 +114,12 @@ void SnapServer::_commit(version_t tid) snapid_t sn = pending_destroy[tid]; dout(7) << "commit " << tid << " destroy " << sn << dendl; snaps.erase(sn); - need_to_purge.insert(sn); + + for (vector<__u32>::const_iterator p = mds->mdsmap->get_data_pg_pools().begin(); + p != mds->mdsmap->get_data_pg_pools().end(); + p++) + need_to_purge[*p].insert(sn); + pending_destroy.erase(tid); } else if (pending_noop.count(tid)) { @@ -157,14 +162,20 @@ void SnapServer::_rollback(version_t tid) void SnapServer::_server_update(bufferlist& bl) { bufferlist::iterator p = bl.begin(); - vector purge; + map > purge; ::decode(purge, p); dout(7) << "_server_update purged " << purge << dendl; - for (vector::iterator p = purge.begin(); + for (map >::iterator p = purge.begin(); p != purge.end(); - p++) - need_to_purge.erase(*p); + p++) { + for (vector::iterator q = p->second.begin(); + q != p->second.end(); + q++) + need_to_purge[p->first].erase(*q); + if (need_to_purge[p->first].empty()) + need_to_purge.erase(p->first); + } version++; } @@ -208,30 +219,36 @@ void SnapServer::check_osd_map(bool force) } dout(10) << "check_osd_map need_to_purge=" << need_to_purge << dendl; - vector purge; - vector purged; + map > all_purge; + map > all_purged; - for (set::iterator p = need_to_purge.begin(); + for (map >::iterator p = need_to_purge.begin(); p != need_to_purge.end(); p++) { - if (mds->osdmap->is_removed_snap(*p)) { - dout(10) << " osdmap marks " << *p << " as removed" << dendl; - purged.push_back(*p); - } else { - purge.push_back(*p); + int id = p->first; + const pg_pool_t& pi = mds->osdmap->get_pg_pool(id); + for (set::iterator q = p->second.begin(); + q != p->second.end(); + q++) { + if (pi.is_removed_snap(*q)) { + dout(10) << " osdmap marks " << *q << " as removed" << dendl; + all_purged[id].push_back(*q); + } else { + all_purge[id].push_back(*q); + } } } - if (purged.size()) { + if (all_purged.size()) { // prepare to remove from need_to_purge list bufferlist bl; - ::encode(purged, bl); + ::encode(all_purged, bl); do_server_update(bl); } - if (!purge.empty()) { - dout(10) << "requesting removal of " << purge << dendl; - MRemoveSnaps *m = new MRemoveSnaps(purge); + if (!all_purge.empty()) { + dout(10) << "requesting removal of " << all_purge << dendl; + MRemoveSnaps *m = new MRemoveSnaps(all_purge); int mon = mds->monmap->pick_mon(); mds->messenger->send_message(m, mds->monmap->get_inst(mon)); } diff --git a/src/mds/SnapServer.h b/src/mds/SnapServer.h index 596cc3c2658b1..7036e38ae54d1 100644 --- a/src/mds/SnapServer.h +++ b/src/mds/SnapServer.h @@ -26,7 +26,7 @@ public: protected: snapid_t last_snap; map snaps; - set need_to_purge; + map > need_to_purge; map pending_create; map pending_destroy; diff --git a/src/messages/MRemoveSnaps.h b/src/messages/MRemoveSnaps.h index a63b07eee6bc6..055cd53e84559 100644 --- a/src/messages/MRemoveSnaps.h +++ b/src/messages/MRemoveSnaps.h @@ -18,11 +18,11 @@ #include "msg/Message.h" struct MRemoveSnaps : public Message { - vector snaps; + map > snaps; MRemoveSnaps() : Message(MSG_REMOVE_SNAPS) { } - MRemoveSnaps(vector& s) : + MRemoveSnaps(map >& s) : Message(MSG_REMOVE_SNAPS) { snaps.swap(s); } diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 78f5759c571d6..0fea3c5bde945 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -610,12 +610,21 @@ bool OSDMonitor::preprocess_remove_snaps(MRemoveSnaps *m) { dout(7) << "preprocess_remove_snaps " << *m << dendl; - for (vector::iterator p = m->snaps.begin(); - p != m->snaps.end(); - p++) { - if (*p > osdmap.max_snap || - !osdmap.removed_snaps.contains(*p)) - return false; + for (map >::iterator q = m->snaps.begin(); + q != m->snaps.end(); + q++) { + if (!osdmap.have_pg_pool(q->first)) { + dout(10) << " ignoring removed_snaps " << q->second << " on non-existant pool " << q->first << dendl; + continue; + } + const pg_pool_t& pi = osdmap.get_pg_pool(q->first); + for (vector::iterator p = q->second.begin(); + p != q->second.end(); + p++) { + if (*p > pi.get_snap_seq() || + !pi.removed_snaps.contains(*p)) + return false; + } } delete m; return true; @@ -625,28 +634,31 @@ bool OSDMonitor::prepare_remove_snaps(MRemoveSnaps *m) { dout(7) << "prepare_remove_snaps " << *m << dendl; - snapid_t max; - for (vector::iterator p = m->snaps.begin(); + for (map >::iterator p = m->snaps.begin(); p != m->snaps.end(); p++) { - if (*p > max) - max = *p; - - if (!osdmap.removed_snaps.contains(*p) && - !pending_inc.removed_snaps.contains(*p)) { - dout(10) << " adding " << *p << " to removed_snaps" << dendl; - pending_inc.removed_snaps.insert(*p); + pg_pool_t& pi = osdmap.pools[p->first]; + for (vector::iterator q = p->second.begin(); + q != p->second.end(); + q++) { + if (!pi.removed_snaps.contains(*q) && + (!pending_inc.new_pools.count(p->first) || + !pending_inc.new_pools[p->first].removed_snaps.contains(*q))) { + if (pending_inc.new_pools.count(p->first) == 0) + pending_inc.new_pools[p->first] = pi; + pg_pool_t& newpi = pending_inc.new_pools[p->first]; + newpi.removed_snaps.insert(*q); + dout(10) << " pool " << p->first << " removed_snaps added " << *q + << " (now " << newpi.removed_snaps << ")" << dendl; + if (*q > newpi.get_snap_seq()) { + dout(10) << " pool " << p->first << " snap_seq " << newpi.get_snap_seq() << " -> " << *q << dendl; + newpi.set_snap_seq(*q); + } + newpi.set_snap_epoch(pending_inc.epoch); + } } } - if (max > osdmap.max_snap && - max > pending_inc.new_max_snap) { - dout(10) << " new_max_snap " << max << dendl; - pending_inc.new_max_snap = max; - } else { - dout(10) << " max_snap " << osdmap.max_snap << " still >= " << max << dendl; - } - delete m; return true; } diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index dcd28cb1091e0..eaf343d2e66ab 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -558,15 +558,53 @@ int OSD::read_superblock() // ====================================================== // PG's +PGPool *OSD::_lookup_pool(int id) +{ + if (pool_map.count(id)) + return pool_map[id]; + return 0; +} + +PGPool* OSD::_get_pool(int id) +{ + PGPool *p = _lookup_pool(id); + if (!p) { + p = new PGPool(id); + pool_map[id] = p; + p->get(); + + const pg_pool_t& pi = osdmap->get_pg_pool(id); + p->info = pi; + p->snapc = pi.get_snap_context(); + } + dout(10) << "_get_pool " << p->id << " " << p->num_pg << " -> " << (p->num_pg+1) << dendl; + p->num_pg++; + return p; +} + +void OSD::_put_pool(int id) +{ + PGPool *p = _lookup_pool(id); + dout(10) << "_put_pool " << id << " " << p->num_pg << " -> " << (p->num_pg-1) << dendl; + p->num_pg--; + if (!p->num_pg) { + pool_map.erase(id); + p->put(); + } +} + + PG *OSD::_open_lock_pg(pg_t pgid, bool no_lockdep_check) { assert(osd_lock.is_locked()); + PGPool *pool = _get_pool(pgid.pool()); + // create PG *pg; sobject_t logoid = make_pg_log_oid(pgid); if (osdmap->get_pg_type(pgid) == CEPH_PG_TYPE_REP) - pg = new ReplicatedPG(this, pgid, logoid); + pg = new ReplicatedPG(this, pool, pgid, logoid); //else if (pgid.is_raid4()) //pg = new RAID4PG(this, pgid); else @@ -696,6 +734,8 @@ void OSD::_remove_unlock_pg(PG *pg) // remove from map pg_map.erase(pgid); + _put_pool(pgid.pool()); + // unlock, and probably delete pg->unlock(); pg->put(); // will delete, if last reference @@ -1876,10 +1916,6 @@ void OSD::handle_osd_map(MOSDMap *m) OSDMap *newmap = new OSDMap; newmap->decode(bl); - // fake inc->removed_snaps - inc.removed_snaps = newmap->get_removed_snaps(); - inc.removed_snaps.subtract(osdmap->get_removed_snaps()); - // kill connections to newly down osds set old; osdmap->get_all_osds(old); @@ -1897,9 +1933,29 @@ void OSD::handle_osd_map(MOSDMap *m) break; } + // update pools + for (map::iterator p = pool_map.begin(); + p != pool_map.end(); + p++) { + const pg_pool_t& pi = osdmap->get_pg_pool(p->first); + if (pi.get_snap_epoch() == cur+1) { + PGPool *pool = p->second; + pool->new_removed_snaps = pi.removed_snaps; + pool->new_removed_snaps.subtract(pool->info.removed_snaps); + dout(10) << " pool " << p->first << " removed_snaps " << pool->info.removed_snaps + << " -> " << pi.removed_snaps + << ", new is " << pool->new_removed_snaps << ")" + << dendl; + pool->info = pi; + pool->snapc = pi.get_snap_context(); + } else { + dout(10) << " pool " << p->first << " unchanged (snap_epoch = " << pi.get_snap_epoch() << ")" << dendl; + } + } + cur++; superblock.current_epoch = cur; - advance_map(t, inc.removed_snaps); + advance_map(t); advanced = true; had_map_since = g_clock.now(); } @@ -1967,13 +2023,12 @@ void OSD::handle_osd_map(MOSDMap *m) * scan placement groups, initiate any replication * activities. */ -void OSD::advance_map(ObjectStore::Transaction& t, interval_set& removed_snaps) +void OSD::advance_map(ObjectStore::Transaction& t) { assert(osd_lock.is_locked()); dout(7) << "advance_map epoch " << osdmap->get_epoch() << " " << pg_map.size() << " pgs" - << " removed_snaps " << removed_snaps << dendl; if (!up_epoch && @@ -2026,15 +2081,15 @@ void OSD::advance_map(ObjectStore::Transaction& t, interval_set& remov pg->lock(); // adjust removed_snaps? - if (!removed_snaps.empty()) { - for (map::iterator p = removed_snaps.m.begin(); - p != removed_snaps.m.end(); + if (!pg->pool->new_removed_snaps.empty()) { + for (map::iterator p = pg->pool->new_removed_snaps.m.begin(); + p != pg->pool->new_removed_snaps.m.end(); p++) for (snapid_t t = 0; t < p->second; ++t) - pg->info.dead_snaps.insert(p->first + t); - dout(10) << *pg << " dead_snaps now " << pg->info.dead_snaps << dendl; + pg->info.snap_trimq.insert(p->first + t); + dout(10) << *pg << " snap_trimq now " << pg->info.snap_trimq << dendl; pg->dirty_info = true; - } + } // no change? if (tacting == pg->acting && (pg->is_active() || !pg->prior_set_affected(osdmap))) { @@ -2184,7 +2239,7 @@ void OSD::activate_map(ObjectStore::Transaction& t) pg->lock(); if (pg->is_active()) { // update started counter - if (!pg->info.dead_snaps.empty()) + if (!pg->info.snap_trimq.empty()) pg->queue_snap_trim(); } else if (pg->is_primary() && diff --git a/src/osd/OSD.h b/src/osd/OSD.h index fde613a14c300..2c4bc9b11ebb8 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -397,7 +397,7 @@ private: void note_down_osd(int osd); void note_up_osd(int osd); - void advance_map(ObjectStore::Transaction& t, interval_set& removed_snaps); + void advance_map(ObjectStore::Transaction& t); void activate_map(ObjectStore::Transaction& t); // osd map cache (past osd maps) @@ -427,9 +427,14 @@ public: protected: // -- placement groups -- + map pool_map; hash_map pg_map; hash_map > waiting_for_pg; + PGPool *_lookup_pool(int id); + PGPool *_get_pool(int id); + void _put_pool(int id); + bool _have_pg(pg_t pgid); PG *_lookup_lock_pg(pg_t pgid); PG *_open_lock_pg(pg_t pg, bool no_lockdep_check=false); // create new PG (in memory) diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 89e91a87752b3..9e74f218988aa 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -25,10 +25,13 @@ void OSDMap::print(ostream& out) << "created " << get_created() << "\n" << "modifed " << get_modified() << "\n" << std::endl; - for (map::iterator p = pools.begin(); p != pools.end(); p++) + for (map::iterator p = pools.begin(); p != pools.end(); p++) { out << "pg_pool " << p->first << " '" << pool_name[p->first] << "' " << p->second << "\n"; + if (!p->second.removed_snaps.empty()) + out << "\tremoved_snaps " << p->second.removed_snaps << "\n"; + } out << std::endl; out << "max_osd " << get_max_osd() << "\n"; @@ -57,10 +60,6 @@ void OSDMap::print(ostream& out) out << "blacklist " << p->first << " expires " << p->second << "\n"; // ignore pg_swap_primary - - out << "max_snap " << get_max_snap() << "\n" - << "removed_snaps " << get_removed_snaps() << "\n" - << std::endl; } void OSDMap::print_summary(ostream& out) diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 37f8258e5ca54..e336053fd7fe9 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -157,9 +157,6 @@ public: map new_blacklist; vector old_blacklist; - snapid_t new_max_snap; - interval_set removed_snaps; - void encode(bufferlist& bl) { // base ::encode(fsid, bl); @@ -183,8 +180,6 @@ public: ::encode(new_lost, bl); ::encode(new_pg_swap_primary, bl); ::encode(old_pg_swap_primary, bl); - ::encode(new_max_snap, bl); - ::encode(removed_snaps.m, bl); ::encode(new_blacklist, bl); ::encode(old_blacklist, bl); } @@ -211,8 +206,6 @@ public: ::decode(new_lost, p); ::decode(new_pg_swap_primary, p); ::decode(old_pg_swap_primary, p); - ::decode(new_max_snap, p); - ::decode(removed_snaps.m, p); ::decode(new_blacklist, p); ::decode(old_blacklist, p); } @@ -246,8 +239,6 @@ private: map pool_name; map name_pool; map pg_swap_primary; // force new osd to be pg primary (if already a member) - snapid_t max_snap; - interval_set removed_snaps; hash_map blacklist; @@ -261,7 +252,7 @@ private: public: OSDMap() : epoch(0), flags(0), - max_osd(0), max_snap(0) { + max_osd(0) { memset(&fsid, 0, sizeof(fsid)); } @@ -284,14 +275,6 @@ private: const utime_t& get_created() const { return created; } const utime_t& get_modified() const { return modified; } - snapid_t get_max_snap() { return max_snap; } - bool is_removed_snap(snapid_t sn) { - if (sn > max_snap) - return false; - return removed_snaps.contains(sn); - } - interval_set& get_removed_snaps() { return removed_snaps; } - bool is_blacklisted(const entity_addr_t& a) { return !blacklist.empty() && blacklist.count(a); } @@ -526,11 +509,6 @@ private: i++) pg_swap_primary.erase(*i); - // snaps - if (inc.new_max_snap > 0) - max_snap = inc.new_max_snap; - removed_snaps.union_of(inc.removed_snaps); - // blacklist for (map::iterator p = inc.new_blacklist.begin(); p != inc.new_blacklist.end(); @@ -579,8 +557,6 @@ private: ::encode(pool_name, blist); ::encode(pg_swap_primary, blist); - ::encode(max_snap, blist); - ::encode(removed_snaps.m, blist); ::encode(blacklist, blist); } @@ -619,8 +595,6 @@ private: ::decode(pg_swap_primary, p); - ::decode(max_snap, p); - ::decode(removed_snaps.m, p); ::decode(blacklist, p); } @@ -805,7 +779,9 @@ private: return -1; } - + bool have_pg_pool(int p) const { + return pools.count(p); + } const pg_pool_t& get_pg_pool(int p) { assert(pools.count(p)); return pools[p]; diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 62f6458faaae3..5f67847105c3b 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -1313,7 +1313,7 @@ void PG::activate(ObjectStore::Transaction& t, // clean up stray objects, snaps clean_up_local(t); - if (!info.dead_snaps.empty()) + if (!info.snap_trimq.empty()) queue_snap_trim(); // init complete pointer diff --git a/src/osd/PG.h b/src/osd/PG.h index df34f80320dbf..91bcc0e19f72f 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -42,6 +42,27 @@ class MOSDSubOp; class MOSDSubOpReply; class MOSDPGInfo; + +struct PGPool { + int id; + atomic_t nref; + int num_pg; + + pg_pool_t info; + SnapContext snapc; // the default pool snapc, ready to go. + + interval_set new_removed_snaps; // newly removed in the last epoch + + PGPool(int i) : id(i), num_pg(0) {} + + void get() { nref.inc(); } + void put() { + if (nref.dec() == 0) + delete this; + } +}; + + /** PG - Replica Placement Group * */ @@ -66,7 +87,7 @@ public: eversion_t log_bottom; // oldest log entry. bool log_backlog; // do we store a complete log? - set dead_snaps; // snaps we need to trim + set snap_trimq; // snaps we need to trim pg_stat_t stats; @@ -117,7 +138,7 @@ public: ::encode(log_backlog, bl); ::encode(stats, bl); history.encode(bl); - ::encode(dead_snaps, bl); + ::encode(snap_trimq, bl); } void decode(bufferlist::iterator &bl) { ::decode(pgid, bl); @@ -127,7 +148,7 @@ public: ::decode(log_backlog, bl); ::decode(stats, bl); history.decode(bl); - ::decode(dead_snaps, bl); + ::decode(snap_trimq, bl); } }; WRITE_CLASS_ENCODER(Info::History) @@ -536,6 +557,7 @@ public: /*** PG ****/ protected: OSD *osd; + PGPool *pool; /** locking and reference counting. * I destroy myself when the reference count hits zero. @@ -760,8 +782,8 @@ public: public: - PG(OSD *o, pg_t p, const sobject_t& oid) : - osd(o), + PG(OSD *o, PGPool *_pool, pg_t p, const sobject_t& oid) : + osd(o), pool(_pool), _lock("PG::_lock"), ref(0), deleted(false), dirty_info(false), dirty_log(false), info(p), log_oid(oid), @@ -776,8 +798,11 @@ public: pg_stats_valid(false), finish_sync_event(NULL) { + pool->get(); + } + virtual ~PG() { + pool->put(); } - virtual ~PG() { } pg_t get_pgid() const { return info.pgid; } int get_nrep() const { return acting.size(); } @@ -975,8 +1000,8 @@ inline ostream& operator<<(ostream& out, const PG& pg) if (lost) out << " l=" << lost; } - if (pg.info.dead_snaps.size()) - out << " dead=" << pg.info.dead_snaps; + if (pg.info.snap_trimq.size()) + out << " snaptrimq=" << pg.info.snap_trimq; out << "]"; diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 80dd2e6d9852d..443e7874c2859 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -663,9 +663,9 @@ bool ReplicatedPG::snap_trimmer() lock(); dout(10) << "snap_trimmer start" << dendl; - while (info.dead_snaps.size() && + while (info.snap_trimq.size() && is_active()) { - snapid_t sn = *info.dead_snaps.begin(); + snapid_t sn = *info.snap_trimq.begin(); coll_t c = info.pgid.to_snap_coll(sn); vector ls; osd->store->collection_list(c, ls); @@ -699,7 +699,7 @@ bool ReplicatedPG::snap_trimmer() // remove snaps vector newsnaps; for (unsigned i=0; iosdmap->is_removed_snap(snaps[i])) + if (!osd->_lookup_pool(info.pgid.pool())->info.is_removed_snap(snaps[i])) newsnaps.push_back(snaps[i]); else { vector::iterator q = snapset.snaps.begin(); @@ -784,7 +784,7 @@ bool ReplicatedPG::snap_trimmer() t.remove_collection(c); osd->store->apply_transaction(t); - info.dead_snaps.erase(sn); + info.snap_trimq.erase(sn); } // done diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 3e3605bb1846d..162c4b119cd25 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -435,8 +435,8 @@ protected: public: - ReplicatedPG(OSD *o, pg_t p, const sobject_t& oid) : - PG(o, p, oid) + ReplicatedPG(OSD *o, PGPool *_pool, pg_t p, const sobject_t& oid) : + PG(o, _pool, p, oid) { } ~ReplicatedPG() {} diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index fbf1f08366159..9ed3ee4852d46 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -435,6 +435,9 @@ struct pg_pool_t { epoch_t get_snap_epoch() const { return v.snap_epoch; } snapid_t get_snap_seq() const { return snapid_t(v.snap_seq); } + void set_snap_seq(snapid_t s) { v.snap_seq = s; } + void set_snap_epoch(epoch_t e) { v.snap_epoch = e; } + bool is_rep() const { return get_type() == CEPH_PG_TYPE_REP; } bool is_raid4() const { return get_type() == CEPH_PG_TYPE_RAID4; } @@ -463,7 +466,7 @@ struct pg_pool_t { lpgp_num_mask = (1 << calc_bits_of(v.lpgp_num-1)) - 1; } - bool is_removed_snap(snapid_t s) { + bool is_removed_snap(snapid_t s) const { if (snaps.size()) return snaps.count(s) == 0; return s <= get_snap_seq() && removed_snaps.contains(s); -- 2.39.5