From: Samuel Just Date: Tue, 10 May 2016 19:12:30 +0000 (-0700) Subject: osd_types: template pg_missing_t based on whether it tracks changes X-Git-Tag: ses5-milestone5~325^2~8 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=abcd1375b193658cc94d973ef71a3e8b740f38e7;p=ceph.git osd_types: template pg_missing_t based on whether it tracks changes We'll use the version with TrackChanges = true as the local missing set so that we can transparently track changes. Also introduce a const interface so that PGBackend methods can abstract over the difference. Finally, switch around type declarations in users. Signed-off-by: Samuel Just --- diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc index cbdd675db099..55046ffe6cea 100644 --- a/src/osd/ECBackend.cc +++ b/src/osd/ECBackend.cc @@ -1475,8 +1475,7 @@ int ECBackend::get_min_avail_to_read_shards( i != miter->second.end(); ++i) { dout(10) << __func__ << ": checking missing_loc " << *i << dendl; - boost::optional m = - get_parent()->maybe_get_shard_missing(*i); + auto m = get_parent()->maybe_get_shard_missing(*i); if (m) { assert(!(*m).is_missing(hoid)); } diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 015e6beedf74..982f6895a1d4 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -5702,9 +5702,9 @@ void OSD::do_command(Connection *con, ceph_tid_t tid, vector& cmd, buffe pg->lock(); fout << *pg << std::endl; - std::map::const_iterator mend = + std::map::const_iterator mend = pg->pg_log.get_missing().get_items().end(); - std::map::const_iterator mi = + std::map::const_iterator mi = pg->pg_log.get_missing().get_items().begin(); for (; mi != mend; ++mi) { fout << mi->first << " -> " << mi->second << std::endl; diff --git a/src/osd/PG.cc b/src/osd/PG.cc index aa94b74c8f5d..8378cdbbd9db 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -337,7 +337,7 @@ void PG::proc_replica_log( dout(10) << " peer osd." << from << " now " << oinfo << " " << omissing << dendl; might_have_unfound.insert(from); - for (map::const_iterator i = + for (map::const_iterator i = omissing.get_items().begin(); i != omissing.get_items().end(); ++i) { @@ -503,7 +503,7 @@ void PG::MissingLoc::add_batch_sources_info( { dout(10) << __func__ << ": adding sources in batch " << sources.size() << dendl; unsigned loop = 0; - for (map::const_iterator i = needs_recovery_map.begin(); + for (map::const_iterator i = needs_recovery_map.begin(); i != needs_recovery_map.end(); ++i) { if (handle && ++loop >= g_conf->osd_loop_before_reset_tphandle) { @@ -525,7 +525,7 @@ bool PG::MissingLoc::add_source_info( bool found_missing = false; unsigned loop = 0; // found items? - for (map::const_iterator p = needs_recovery_map.begin(); + for (map::const_iterator p = needs_recovery_map.begin(); p != needs_recovery_map.end(); ++p) { const hobject_t &soid(p->first); @@ -581,7 +581,7 @@ bool PG::MissingLoc::add_source_info( void PG::discover_all_missing(map > &query_map) { - const pg_missing_t &missing = pg_log.get_missing(); + auto &missing = pg_log.get_missing(); assert(have_unfound()); dout(10) << __func__ << " " @@ -642,7 +642,7 @@ bool PG::needs_recovery() const { assert(is_primary()); - const pg_missing_t &missing = pg_log.get_missing(); + auto &missing = pg_log.get_missing(); if (missing.num_missing()) { dout(10) << __func__ << " primary has " << missing.num_missing() @@ -1580,7 +1580,7 @@ void PG::activate(ObjectStore::Transaction& t, info.last_epoch_started = activation_epoch; } - const pg_missing_t &missing = pg_log.get_missing(); + auto &missing = pg_log.get_missing(); if (is_primary()) { last_update_ondisk = info.last_update; diff --git a/src/osd/PG.h b/src/osd/PG.h index cc196430e38f..b1c661644169 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -321,7 +321,7 @@ public: ghobject_t pgmeta_oid; class MissingLoc { - map needs_recovery_map; + map needs_recovery_map; map, hobject_t::BitwiseComparator > missing_loc; set missing_loc_sources; PG *pg; @@ -341,7 +341,7 @@ public: bool needs_recovery( const hobject_t &hoid, eversion_t *v = 0) const { - map::const_iterator i = + map::const_iterator i = needs_recovery_map.find(hoid); if (i == needs_recovery_map.end()) return false; @@ -359,7 +359,7 @@ public: const set &acting) const; uint64_t num_unfound() const { uint64_t ret = 0; - for (map::const_iterator i = + for (map::const_iterator i = needs_recovery_map.begin(); i != needs_recovery_map.end(); ++i) { @@ -382,11 +382,11 @@ public: missing_loc[hoid].erase(location); } void add_active_missing(const pg_missing_t &missing) { - for (map::const_iterator i = + for (map::const_iterator i = missing.get_items().begin(); i != missing.get_items().end(); ++i) { - map::const_iterator j = + map::const_iterator j = needs_recovery_map.find(i->first); if (j == needs_recovery_map.end()) { needs_recovery_map.insert(*i); @@ -397,7 +397,7 @@ public: } void add_missing(const hobject_t &hoid, eversion_t need, eversion_t have) { - needs_recovery_map[hoid] = pg_missing_t::item(need, have); + needs_recovery_map[hoid] = pg_missing_item(need, have); } void revise_need(const hobject_t &hoid, eversion_t need) { assert(needs_recovery(hoid)); @@ -439,7 +439,7 @@ public: const map &pmissing, const map &pinfo) { recovered(hoid); - boost::optional item; + boost::optional item; auto miter = missing.get_items().find(hoid); if (miter != missing.get_items().end()) { item = miter->second; @@ -483,7 +483,7 @@ public: const map, hobject_t::BitwiseComparator> &get_missing_locs() const { return missing_loc; } - const map &get_needs_recovery() const { + const map &get_needs_recovery() const { return needs_recovery_map; } } missing_loc; diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h index 228db93a9263..03a464990ae6 100644 --- a/src/osd/PGBackend.h +++ b/src/osd/PGBackend.h @@ -131,10 +131,10 @@ typedef ceph::shared_ptr OSDMapRef; virtual const map, hobject_t::BitwiseComparator> &get_missing_loc_shards() const = 0; - virtual const pg_missing_t &get_local_missing() const = 0; + virtual const pg_missing_tracker_t &get_local_missing() const = 0; virtual const map &get_shard_missing() const = 0; - virtual boost::optional maybe_get_shard_missing( + virtual boost::optional maybe_get_shard_missing( pg_shard_t peer) const { if (peer == primary_shard()) { return get_local_missing(); @@ -142,14 +142,14 @@ typedef ceph::shared_ptr OSDMapRef; map::const_iterator i = get_shard_missing().find(peer); if (i == get_shard_missing().end()) { - return boost::optional(); + return boost::optional(); } else { return i->second; } } } - virtual const pg_missing_t &get_shard_missing(pg_shard_t peer) const { - boost::optional m = maybe_get_shard_missing(peer); + virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const { + auto m = maybe_get_shard_missing(peer); assert(m); return *m; } diff --git a/src/osd/PGLog.cc b/src/osd/PGLog.cc index 7338b9835559..a439bb165c3c 100644 --- a/src/osd/PGLog.cc +++ b/src/osd/PGLog.cc @@ -216,7 +216,7 @@ void PGLog::proc_replica_log( we will send the peer enough log to arrive at the same state. */ - for (map::const_iterator i = omissing.get_items().begin(); + for (map::const_iterator i = omissing.get_items().begin(); i != omissing.get_items().end(); ++i) { dout(20) << " before missing " << i->first << " need " << i->second.need @@ -394,7 +394,7 @@ void PGLog::merge_log(ObjectStore::Transaction& t, // The logs must overlap. assert(log.head >= olog.tail && olog.head >= log.tail); - for (map::const_iterator i = missing.get_items().begin(); + for (map::const_iterator i = missing.get_items().begin(); i != missing.get_items().end(); ++i) { dout(20) << "pg_missing_t sobject: " << i->first << dendl; diff --git a/src/osd/PGLog.h b/src/osd/PGLog.h index c2fbbd582b70..7f27517af523 100644 --- a/src/osd/PGLog.h +++ b/src/osd/PGLog.h @@ -426,7 +426,8 @@ protected: //////////////////// data members //////////////////// map divergent_priors; - pg_missing_t missing; + + pg_missing_tracker_t missing; IndexedLog log; eversion_t dirty_to; ///< must clear/writeout all keys <= dirty_to @@ -515,7 +516,7 @@ public: //////////////////// get or set missing //////////////////// - const pg_missing_t& get_missing() const { return missing; } + const pg_missing_tracker_t& get_missing() const { return missing; } void resort_missing(bool sort_bitwise) { missing.resort(sort_bitwise); } diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 478da38c6220..930e534eecb9 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -803,9 +803,9 @@ int ReplicatedPG::do_command( } f->dump_int("num_missing", missing.num_missing()); f->dump_int("num_unfound", get_num_unfound()); - const map &needs_recovery_map = + const map &needs_recovery_map = missing_loc.get_needs_recovery(); - map::const_iterator p = + map::const_iterator p = needs_recovery_map.upper_bound(offset); { f->open_array_section("objects"); @@ -962,7 +962,7 @@ void ReplicatedPG::do_pg_op(OpRequestRef op) // ensure sort order is correct pg_log.resort_missing(get_sort_bitwise()); - map::const_iterator missing_iter = + map::const_iterator missing_iter = pg_log.get_missing().get_items().lower_bound(current); vector::iterator ls_iter = sentries.begin(); hobject_t _max = hobject_t::get_max(); @@ -1149,7 +1149,7 @@ void ReplicatedPG::do_pg_op(OpRequestRef op) // ensure sort order is correct pg_log.resort_missing(get_sort_bitwise()); - map::const_iterator missing_iter = + map::const_iterator missing_iter = pg_log.get_missing().get_items().lower_bound(current); vector::iterator ls_iter = sentries.begin(); hobject_t _max = hobject_t::get_max(); @@ -9815,9 +9815,9 @@ void ReplicatedPG::mark_all_unfound_lost( list log_entries; utime_t mtime = ceph_clock_now(cct); - map::const_iterator m = + map::const_iterator m = missing_loc.get_needs_recovery().begin(); - map::const_iterator mend = + map::const_iterator mend = missing_loc.get_needs_recovery().end(); ObcLockManager manager; @@ -10551,7 +10551,7 @@ uint64_t ReplicatedPG::recover_primary(uint64_t max, ThreadPool::TPHandle &handl latest = 0; soid = p->second; } - const pg_missing_t::item& item = missing.get_items().find(p->second)->second; + const pg_missing_item& item = missing.get_items().find(p->second)->second; ++p; hobject_t head = soid; @@ -10805,7 +10805,7 @@ uint64_t ReplicatedPG::recover_replicas(uint64_t max, ThreadPool::TPHandle &hand } dout(10) << __func__ << ": recover_object_replicas(" << soid << ")" << dendl; - map::const_iterator r = m.get_items().find(soid); + map::const_iterator r = m.get_items().find(soid); started += prep_object_replica_pushes(soid, r->second.need, h); } diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index faf85ffb2176..3fe75339b54b 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -352,7 +352,7 @@ public: return peer_info; } using PGBackend::Listener::get_shard_info; - const pg_missing_t &get_local_missing() const { + const pg_missing_tracker_t &get_local_missing() const { return pg_log.get_missing(); } const PGLog &get_log() const { diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 8cb98af1fff9..79c5db829e9a 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -3763,77 +3763,9 @@ ostream& pg_log_t::print(ostream& out) const return out; } - // -- pg_missing_t -- -void pg_missing_t::resort(bool sort_bitwise) -{ - if (missing.key_comp().bitwise != sort_bitwise) { - map tmp; - tmp.swap(missing); - missing = map( - hobject_t::ComparatorWithDefault(sort_bitwise)); - missing.insert(tmp.begin(), tmp.end()); - } -} - -void pg_missing_t::encode(bufferlist &bl) const -{ - ENCODE_START(3, 2, bl); - ::encode(missing, bl); - ENCODE_FINISH(bl); -} - -void pg_missing_t::decode(bufferlist::iterator &bl, int64_t pool) -{ - DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl); - ::decode(missing, bl); - DECODE_FINISH(bl); - - if (struct_v < 3) { - // Handle hobject_t upgrade - map tmp; - for (map::iterator i = missing.begin(); - i != missing.end(); - ) { - if (!i->first.is_max() && i->first.pool == -1) { - hobject_t to_insert(i->first); - to_insert.pool = pool; - tmp[to_insert] = i->second; - missing.erase(i++); - } else { - ++i; - } - } - missing.insert(tmp.begin(), tmp.end()); - } - - for (map::iterator it = missing.begin(); - it != missing.end(); - ++it) - rmissing[it->second.need.version] = it->first; -} - -void pg_missing_t::dump(Formatter *f) const -{ - f->open_array_section("missing"); - for (map::const_iterator p = missing.begin(); p != missing.end(); ++p) { - f->open_object_section("item"); - f->dump_stream("object") << p->first; - p->second.dump(f); - f->close_section(); - } - f->close_section(); -} - -void pg_missing_t::generate_test_instances(list& o) -{ - o.push_back(new pg_missing_t); - o.push_back(new pg_missing_t); - o.back()->add(hobject_t(object_t("foo"), "foo", 123, 456, 0, ""), eversion_t(5, 6), eversion_t(5, 1)); -} - -ostream& operator<<(ostream& out, const pg_missing_t::item& i) +ostream& operator<<(ostream& out, const pg_missing_item& i) { out << i.need; if (i.have != eversion_t()) @@ -3841,160 +3773,6 @@ ostream& operator<<(ostream& out, const pg_missing_t::item& i) return out; } -ostream& operator<<(ostream& out, const pg_missing_t& missing) -{ - out << "missing(" << missing.num_missing(); - //if (missing.num_lost()) out << ", " << missing.num_lost() << " lost"; - out << ")"; - return out; -} - - -unsigned int pg_missing_t::num_missing() const -{ - return missing.size(); -} - -bool pg_missing_t::have_missing() const -{ - return !missing.empty(); -} - -void pg_missing_t::swap(pg_missing_t& o) -{ - missing.swap(o.missing); - rmissing.swap(o.rmissing); -} - -bool pg_missing_t::is_missing(const hobject_t& oid) const -{ - return (missing.find(oid) != missing.end()); -} - -bool pg_missing_t::is_missing(const hobject_t& oid, eversion_t v) const -{ - map::const_iterator m = missing.find(oid); - if (m == missing.end()) - return false; - const pg_missing_t::item &item(m->second); - if (item.need > v) - return false; - return true; -} - -eversion_t pg_missing_t::have_old(const hobject_t& oid) const -{ - map::const_iterator m = missing.find(oid); - if (m == missing.end()) - return eversion_t(); - const pg_missing_t::item &item(m->second); - return item.have; -} - -/* - * this needs to be called in log order as we extend the log. it - * assumes missing is accurate up through the previous log entry. - */ -void pg_missing_t::add_next_event(const pg_log_entry_t& e) -{ - if (e.is_update()) { - map::iterator missing_it; - missing_it = missing.find(e.soid); - bool is_missing_divergent_item = missing_it != missing.end(); - if (e.prior_version == eversion_t() || e.is_clone()) { - // new object. - if (is_missing_divergent_item) { // use iterator - rmissing.erase((missing_it->second).need.version); - missing_it->second = item(e.version, eversion_t()); // .have = nil - } else // create new element in missing map - missing[e.soid] = item(e.version, eversion_t()); // .have = nil - } else if (is_missing_divergent_item) { - // already missing (prior). - rmissing.erase((missing_it->second).need.version); - (missing_it->second).need = e.version; // leave .have unchanged. - } else if (e.is_backlog()) { - // May not have prior version - assert(0 == "these don't exist anymore"); - } else { - // not missing, we must have prior_version (if any) - assert(!is_missing_divergent_item); - missing[e.soid] = item(e.version, e.prior_version); - } - rmissing[e.version.version] = e.soid; - } else if (e.is_delete()) { - rm(e.soid, e.version); - } -} - -void pg_missing_t::revise_need(hobject_t oid, eversion_t need) -{ - if (missing.count(oid)) { - rmissing.erase(missing[oid].need.version); - missing[oid].need = need; // no not adjust .have - } else { - missing[oid] = item(need, eversion_t()); - } - rmissing[need.version] = oid; -} - -void pg_missing_t::revise_have(hobject_t oid, eversion_t have) -{ - if (missing.count(oid)) { - missing[oid].have = have; - } -} - -void pg_missing_t::add(const hobject_t& oid, eversion_t need, eversion_t have) -{ - missing[oid] = item(need, have); - rmissing[need.version] = oid; -} - -void pg_missing_t::rm(const hobject_t& oid, eversion_t v) -{ - std::map::iterator p = missing.find(oid); - if (p != missing.end() && p->second.need <= v) - rm(p); -} - -void pg_missing_t::rm(std::map::const_iterator m) -{ - rmissing.erase(m->second.need.version); - missing.erase(m); -} - -void pg_missing_t::got(const hobject_t& oid, eversion_t v) -{ - std::map::iterator p = missing.find(oid); - assert(p != missing.end()); - assert(p->second.need <= v); - got(p); -} - -void pg_missing_t::got(std::map::iterator m) -{ - rmissing.erase(m->second.need.version); - missing.erase(m); -} - -void pg_missing_t::split_into( - pg_t child_pgid, - unsigned split_bits, - pg_missing_t *omissing) -{ - unsigned mask = ~((~0)<::iterator i = missing.begin(); - i != missing.end(); - ) { - if ((i->first.get_hash() & mask) == child_pgid.m_seed) { - omissing->add(i->first, i->second.need, i->second.have); - rm(i++); - } else { - ++i; - } - } -} - // -- object_copy_cursor_t -- void object_copy_cursor_t::encode(bufferlist& bl) const diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 169ee2390a14..ddc133a90c7b 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -2744,80 +2744,360 @@ inline ostream& operator<<(ostream& out, const pg_log_t& log) * kept in memory, as a supplement to pg_log_t * also used to pass missing info in messages. */ -class pg_missing_t { +struct pg_missing_item { + eversion_t need, have; + pg_missing_item() {} + explicit pg_missing_item(eversion_t n) : need(n) {} // have no old version + pg_missing_item(eversion_t n, eversion_t h) : need(n), have(h) {} + + void encode(bufferlist& bl) const { + ::encode(need, bl); + ::encode(have, bl); + } + void decode(bufferlist::iterator& bl) { + ::decode(need, bl); + ::decode(have, bl); + } + void dump(Formatter *f) const { + f->dump_stream("need") << need; + f->dump_stream("have") << have; + } + static void generate_test_instances(list& o) { + o.push_back(new pg_missing_item); + o.push_back(new pg_missing_item); + o.back()->need = eversion_t(1, 2); + o.back()->have = eversion_t(1, 1); + } + bool operator==(const pg_missing_item &rhs) const { + return need == rhs.need && have == rhs.have; + } + bool operator!=(const pg_missing_item &rhs) const { + return !(*this == rhs); + } +}; +WRITE_CLASS_ENCODER(pg_missing_item) +ostream& operator<<(ostream& out, const pg_missing_item &item); + +class pg_missing_const_i { public: - struct item { - eversion_t need, have; - item() {} - explicit item(eversion_t n) : need(n) {} // have no old version - item(eversion_t n, eversion_t h) : need(n), have(h) {} - - void encode(bufferlist& bl) const { - ::encode(need, bl); - ::encode(have, bl); - } - void decode(bufferlist::iterator& bl) { - ::decode(need, bl); - ::decode(have, bl); - } - void dump(Formatter *f) const { - f->dump_stream("need") << need; - f->dump_stream("have") << have; - } - static void generate_test_instances(list& o) { - o.push_back(new item); - o.push_back(new item); - o.back()->need = eversion_t(1, 2); - o.back()->have = eversion_t(1, 1); + virtual const map & + get_items() const = 0; + virtual const map &get_rmissing() const = 0; + virtual unsigned int num_missing() const = 0; + virtual bool have_missing() const = 0; + virtual bool is_missing(const hobject_t& oid) const = 0; + virtual bool is_missing(const hobject_t& oid, eversion_t v) const = 0; + virtual eversion_t have_old(const hobject_t& oid) const = 0; + virtual ~pg_missing_const_i() {} +}; + + +template +class ChangeTracker { +public: + void changed(const hobject_t &obj) {} + template + void get_changed(F &&f) const {} + void flush() {} + void clean() { + return true; + } +}; +template <> +class ChangeTracker { + set _changed; +public: + void changed(const hobject_t &obj) { + _changed.insert(obj); + } + template + void get_changed(F &&f) { + for (auto const &i: _changed) { + f(i); } - }; - WRITE_CLASS_ENCODER(item) + } + void flush() { + _changed.clear(); + } + bool clean() const { + return _changed.empty(); + } +}; -private: +template +class pg_missing_set : public pg_missing_const_i { + using item = pg_missing_item; map missing; // oid -> (need v, have v) map rmissing; // v -> oid + ChangeTracker tracker; public: - const map &get_items() const { + pg_missing_set() = default; + + template + pg_missing_set(const missing_type &m) : + missing(m.get_items()), rmissing(m.get_rmissing()) { + static_assert( + !TrackChanges, + "Cannot copy construct into tracker pg_missing object"); + } + + const map &get_items() const override { return missing; } - const map &get_rmissing() const { + const map &get_rmissing() const override { return rmissing; } - unsigned int num_missing() const; - bool have_missing() const; - bool is_missing(const hobject_t& oid) const; - bool is_missing(const hobject_t& oid, eversion_t v) const; - eversion_t have_old(const hobject_t& oid) const; - - void swap(pg_missing_t& o); - void add_next_event(const pg_log_entry_t& e); - void revise_need(hobject_t oid, eversion_t need); - void revise_have(hobject_t oid, eversion_t have); - void add(const hobject_t& oid, eversion_t need, eversion_t have); - void rm(const hobject_t& oid, eversion_t v); - void rm(std::map::const_iterator m); - void got(const hobject_t& oid, eversion_t v); - void got(std::map::const_iterator m); - void split_into(pg_t child_pgid, unsigned split_bits, pg_missing_t *omissing); + unsigned int num_missing() const override { + return missing.size(); + } + bool have_missing() const override { + return !missing.empty(); + } + bool is_missing(const hobject_t& oid) const override { + return (missing.find(oid) != missing.end()); + } + bool is_missing(const hobject_t& oid, eversion_t v) const override { + map::const_iterator m = + missing.find(oid); + if (m == missing.end()) + return false; + const item &item(m->second); + if (item.need > v) + return false; + return true; + } + eversion_t have_old(const hobject_t& oid) const override { + map::const_iterator m = + missing.find(oid); + if (m == missing.end()) + return eversion_t(); + const item &item(m->second); + return item.have; + } + + void swap(pg_missing_set& o) { + for (auto &&i: missing) + tracker.changed(i.first); + missing.swap(o.missing); + rmissing.swap(o.rmissing); + for (auto &&i: missing) + tracker.changed(i.first); + } + + /* + * this needs to be called in log order as we extend the log. it + * assumes missing is accurate up through the previous log entry. + */ + void add_next_event(const pg_log_entry_t& e) { + if (e.is_update()) { + map::iterator missing_it; + missing_it = missing.find(e.soid); + bool is_missing_divergent_item = missing_it != missing.end(); + if (e.prior_version == eversion_t() || e.is_clone()) { + // new object. + if (is_missing_divergent_item) { // use iterator + rmissing.erase((missing_it->second).need.version); + missing_it->second = item(e.version, eversion_t()); // .have = nil + } else // create new element in missing map + missing[e.soid] = item(e.version, eversion_t()); // .have = nil + } else if (is_missing_divergent_item) { + // already missing (prior). + rmissing.erase((missing_it->second).need.version); + (missing_it->second).need = e.version; // leave .have unchanged. + } else if (e.is_backlog()) { + // May not have prior version + assert(0 == "these don't exist anymore"); + } else { + // not missing, we must have prior_version (if any) + assert(!is_missing_divergent_item); + missing[e.soid] = item(e.version, e.prior_version); + } + rmissing[e.version.version] = e.soid; + } else if (e.is_delete()) { + rm(e.soid, e.version); + } + + tracker.changed(e.soid); + } + + void revise_need(hobject_t oid, eversion_t need) { + if (missing.count(oid)) { + rmissing.erase(missing[oid].need.version); + missing[oid].need = need; // no not adjust .have + } else { + missing[oid] = item(need, eversion_t()); + } + rmissing[need.version] = oid; + + tracker.changed(oid); + } + + void revise_have(hobject_t oid, eversion_t have) { + if (missing.count(oid)) { + tracker.changed(oid); + missing[oid].have = have; + } + } + + void add(const hobject_t& oid, eversion_t need, eversion_t have) { + missing[oid] = item(need, have); + rmissing[need.version] = oid; + tracker.changed(oid); + } + + void rm(const hobject_t& oid, eversion_t v) { + std::map::iterator p = missing.find(oid); + if (p != missing.end() && p->second.need <= v) + rm(p); + } + + void rm(std::map::const_iterator m) { + tracker.changed(m->first); + rmissing.erase(m->second.need.version); + missing.erase(m); + } + + void got(const hobject_t& oid, eversion_t v) { + std::map::iterator p = missing.find(oid); + assert(p != missing.end()); + assert(p->second.need <= v); + got(p); + } + + void got(std::map::const_iterator m) { + tracker.changed(m->first); + rmissing.erase(m->second.need.version); + missing.erase(m); + } + + void split_into( + pg_t child_pgid, + unsigned split_bits, + pg_missing_set *omissing) { + unsigned mask = ~((~0)<::iterator i = missing.begin(); + i != missing.end(); + ) { + if ((i->first.get_hash() & mask) == child_pgid.m_seed) { + omissing->add(i->first, i->second.need, i->second.have); + rm(i++); + } else { + ++i; + } + } + } void clear() { + for (auto const &i: missing) + tracker.changed(i.first); missing.clear(); rmissing.clear(); } - void resort(bool sort_bitwise); + void resort(bool sort_bitwise) { + if (missing.key_comp().bitwise != sort_bitwise) { + map tmp; + tmp.swap(missing); + missing = map( + hobject_t::ComparatorWithDefault(sort_bitwise)); + missing.insert(tmp.begin(), tmp.end()); + } + } - void encode(bufferlist &bl) const; - void decode(bufferlist::iterator &bl, int64_t pool = -1); - void dump(Formatter *f) const; - static void generate_test_instances(list& o); + void encode(bufferlist &bl) const { + ENCODE_START(3, 2, bl); + ::encode(missing, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator &bl, int64_t pool = -1) { + for (auto const &i: missing) + tracker.changed(i.first); + DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl); + ::decode(missing, bl); + DECODE_FINISH(bl); + + if (struct_v < 3) { + // Handle hobject_t upgrade + map tmp; + for (map::iterator i = + missing.begin(); + i != missing.end(); + ) { + if (!i->first.is_max() && i->first.pool == -1) { + hobject_t to_insert(i->first); + to_insert.pool = pool; + tmp[to_insert] = i->second; + missing.erase(i++); + } else { + ++i; + } + } + missing.insert(tmp.begin(), tmp.end()); + } + + for (map::iterator it = + missing.begin(); + it != missing.end(); + ++it) + rmissing[it->second.need.version] = it->first; + for (auto const &i: missing) + tracker.changed(i.first); + } + void dump(Formatter *f) const { + f->open_array_section("missing"); + for (map::const_iterator p = + missing.begin(); p != missing.end(); ++p) { + f->open_object_section("item"); + f->dump_stream("object") << p->first; + p->second.dump(f); + f->close_section(); + } + f->close_section(); + } + static void generate_test_instances(list& o) { + o.push_back(new pg_missing_set); + o.push_back(new pg_missing_set); + o.back()->add( + hobject_t(object_t("foo"), "foo", 123, 456, 0, ""), + eversion_t(5, 6), eversion_t(5, 1)); + } + template + std::enable_if get_changed (F &&f) const { + tracker.get_changed(f); + } + template + std::enable_if flush() { + tracker.flush(); + } + template + std::enable_if clean() const { + return tracker.clean(); + } }; -WRITE_CLASS_ENCODER(pg_missing_t::item) -WRITE_CLASS_ENCODER(pg_missing_t) +template +void encode( + const pg_missing_set &c, bufferlist &bl, uint64_t features=0) { + ENCODE_DUMP_PRE(); + c.encode(bl); + ENCODE_DUMP_POST(cl); +} +template +void decode(pg_missing_set &c, bufferlist::iterator &p) { + c.decode(p); +} +template +ostream& operator<<(ostream& out, const pg_missing_set &missing) +{ + out << "missing(" << missing.num_missing(); + //if (missing.num_lost()) out << ", " << missing.num_lost() << " lost"; + out << ")"; + return out; +} + +using pg_missing_t = pg_missing_set; +using pg_missing_tracker_t = pg_missing_set; -ostream& operator<<(ostream& out, const pg_missing_t::item& i); -ostream& operator<<(ostream& out, const pg_missing_t& missing); /** * pg list objects response format diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h index e39641111f09..921ac6b11b2d 100644 --- a/src/test/encoding/types.h +++ b/src/test/encoding/types.h @@ -70,7 +70,7 @@ TYPE(pg_interval_t) TYPE_FEATUREFUL(pg_query_t) TYPE(pg_log_entry_t) TYPE(pg_log_t) -TYPE(pg_missing_t::item) +TYPE(pg_missing_item) TYPE(pg_missing_t) TYPE(pg_ls_response_t) TYPE(pg_nls_response_t) diff --git a/src/test/osd/types.cc b/src/test/osd/types.cc index 3d24d9823698..d4036a6db696 100644 --- a/src/test/osd/types.cc +++ b/src/test/osd/types.cc @@ -1049,7 +1049,7 @@ TEST(pg_missing_t, rm) missing.rm(oid, eversion_t(epoch * 2,20)); EXPECT_FALSE(missing.is_missing(oid)); } - // void pg_missing_t::rm(const std::map::iterator &m) + // void pg_missing_t::rm(const std::map::iterator &m) { hobject_t oid(object_t("objname"), "key", 123, 456, 0, ""); pg_missing_t missing; @@ -1081,7 +1081,7 @@ TEST(pg_missing_t, got) missing.got(oid, eversion_t(epoch * 2,20)); EXPECT_FALSE(missing.is_missing(oid)); } - // void pg_missing_t::got(const std::map::iterator &m) + // void pg_missing_t::got(const std::map::iterator &m) { hobject_t oid(object_t("objname"), "key", 123, 456, 0, ""); pg_missing_t missing;