From 17d7a24d6151948866e3ce893f56a46af0a9d401 Mon Sep 17 00:00:00 2001 From: "sjust@redhat.com" Date: Mon, 8 Apr 2019 16:32:10 -0700 Subject: [PATCH] osd/: move pg_log, missing_loc mutations to PeeringState Signed-off-by: Samuel Just --- src/osd/PG.cc | 28 ++++--- src/osd/PG.h | 7 +- src/osd/PeeringState.cc | 12 +++ src/osd/PeeringState.h | 16 ++++ src/osd/PrimaryLogPG.cc | 173 ++++++++++++++++++++-------------------- src/osd/PrimaryLogPG.h | 12 +-- 6 files changed, 140 insertions(+), 108 deletions(-) diff --git a/src/osd/PG.cc b/src/osd/PG.cc index aef2bf308f711..0252b53111143 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -187,8 +187,6 @@ PG::PG(OSDService *o, OSDMapRef curmap, this), pg_whoami(recovery_state.pg_whoami), info(recovery_state.info), - pg_log(recovery_state.pg_log), - missing_loc(recovery_state.missing_loc), pg_id(p), coll(p), osd(o), @@ -805,7 +803,8 @@ bool PG::check_in_progress_op( { return ( projected_log.get_request(r, version, user_version, return_code) || - pg_log.get_log().get_request(r, version, user_version, return_code)); + recovery_state.get_pg_log().get_log().get_request( + r, version, user_version, return_code)); } void PG::publish_stats_to_osd() @@ -1620,7 +1619,6 @@ void PG::on_new_interval() { scrub_queued = false; projected_last_update = eversion_t(); cancel_recovery(); - plpg_on_new_interval(); } epoch_t PG::oldest_stored_osdmap() { @@ -1845,6 +1843,14 @@ void PG::send_pg_created(pg_t pgid) osd->send_pg_created(pgid); } +void PG::rebuild_missing_set_with_deletes(PGLog &pglog) +{ + pglog.rebuild_missing_set_with_deletes( + osd->store, + ch, + recovery_state.get_info()); +} + void PG::on_activate_committed() { if (!is_primary()) { @@ -2781,8 +2787,8 @@ void PG::chunky_scrub(ThreadPool::TPHandle &handle) } if (scrubber.subset_last_update == eversion_t()) { for (list::const_reverse_iterator p = - pg_log.get_log().log.rbegin(); - p != pg_log.get_log().log.rend(); + recovery_state.get_pg_log().get_log().log.rbegin(); + p != recovery_state.get_pg_log().get_log().log.rend(); ++p) { if (p->soid >= scrubber.start && p->soid < scrubber.end) { @@ -3912,14 +3918,16 @@ void PG::dump_pgstate_history(Formatter *f) void PG::dump_missing(Formatter *f) { - for (auto& i : pg_log.get_missing().get_items()) { + for (auto& i : recovery_state.get_pg_log().get_missing().get_items()) { f->open_object_section("object"); f->dump_object("oid", i.first); f->dump_object("missing_info", i.second); - if (missing_loc.needs_recovery(i.first)) { - f->dump_bool("unfound", missing_loc.is_unfound(i.first)); + if (recovery_state.get_missing_loc().needs_recovery(i.first)) { + f->dump_bool( + "unfound", + recovery_state.get_missing_loc().is_unfound(i.first)); f->open_array_section("locations"); - for (auto l : missing_loc.get_locations(i.first)) { + for (auto l : recovery_state.get_missing_loc().get_locations(i.first)) { f->dump_object("shard", l); } f->close_section(); diff --git a/src/osd/PG.h b/src/osd/PG.h index c51cdc275ef71..b45de5876e301 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -184,8 +184,6 @@ protected: */ pg_shard_t pg_whoami; pg_info_t &info; - PGLog &pg_log; - MissingLoc &missing_loc; public: // -- members -- @@ -397,7 +395,6 @@ public: void clear_want_pg_temp() override; void on_new_interval() override; - virtual void plpg_on_new_interval() = 0; void on_role_change() override; virtual void plpg_on_role_change() = 0; @@ -486,6 +483,8 @@ public: void send_pg_created(pg_t pgid) override; + void rebuild_missing_set_with_deletes(PGLog &pglog) override; + void queue_peering_event(PGPeeringEventRef evt); void do_peering_event(PGPeeringEventRef evt, PeeringCtx *rcx); void queue_null(epoch_t msg_epoch, epoch_t query_epoch); @@ -1446,7 +1445,7 @@ protected: get_osdmap_epoch(), projected_last_update.version+1); ceph_assert(at_version > info.last_update); - ceph_assert(at_version > pg_log.get_head()); + ceph_assert(at_version > recovery_state.get_pg_log().get_head()); ceph_assert(at_version > projected_last_update); return at_version; } diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index 58937df0f7403..51a2189426786 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -721,6 +721,17 @@ void PeeringState::on_new_interval() upacting_features &= osdmap->get_xinfo(*p).features; } + psdout(20) << __func__ << " checking missing set deletes flag. missing = " + << get_pg_log().get_missing() << dendl; + + if (!pg_log.get_missing().may_include_deletes && + get_osdmap()->test_flag(CEPH_OSDMAP_RECOVERY_DELETES)) { + pl->rebuild_missing_set_with_deletes(pg_log); + } + ceph_assert( + pg_log.get_missing().may_include_deletes == get_osdmap()->test_flag( + CEPH_OSDMAP_RECOVERY_DELETES)); + pl->on_new_interval(); } @@ -799,6 +810,7 @@ void PeeringState::clear_primary_state() clear_recovery_state(); last_update_ondisk = eversion_t(); + missing_loc.clear(); pl->clear_primary_state(); } diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h index a00402adeb664..4cb56a5743a45 100644 --- a/src/osd/PeeringState.h +++ b/src/osd/PeeringState.h @@ -163,6 +163,9 @@ public: virtual epoch_t oldest_stored_osdmap() = 0; virtual LogChannel &get_clog() = 0; + // On disk representation changes + virtual void rebuild_missing_set_with_deletes(PGLog &pglog) = 0; + // Logging virtual PerfCounters &get_peering_perf() = 0; virtual PerfCounters &get_perf_logger() = 0; @@ -1528,6 +1531,10 @@ public: bool transaction_applied, bool async); + void add_local_next_event(const pg_log_entry_t& e) { + pg_log.missing_add_next_entry(e); + } + void update_trim_to() { bool hard_limit = (get_osdmap()->test_flag(CEPH_OSDMAP_PGLOG_HARDLIMIT)); if (hard_limit) @@ -1555,6 +1562,11 @@ public: pg_shard_t peer, const hobject_t soid); + void object_recovered( + const hobject_t &hoid) { + missing_loc.recovered(hoid); + } + void update_backfill_progress( const hobject_t &updated_backfill, const pg_stat_t &updated_stats, @@ -1591,6 +1603,10 @@ public: void recovery_committed_to(eversion_t version); + void set_last_requested(version_t v) { + pg_log.set_last_requested(v); + } + void complete_write(eversion_t v, eversion_t lc); void local_write_applied(eversion_t v) { last_update_applied = v; diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index e2ce5486ba0c1..a45a0f7dbfdb2 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -372,10 +372,10 @@ void PrimaryLogPG::on_local_recover( derr << __func__ << " " << hoid << " had no clone_snaps" << dendl; } } - if (!is_delete && pg_log.get_missing().is_missing(recovery_info.soid) && - pg_log.get_missing().get_items().find(recovery_info.soid)->second.need > recovery_info.version) { + if (!is_delete && recovery_state.get_pg_log().get_missing().is_missing(recovery_info.soid) && + recovery_state.get_pg_log().get_missing().get_items().find(recovery_info.soid)->second.need > recovery_info.version) { ceph_assert(is_primary()); - const pg_log_entry_t *latest = pg_log.get_log().objects.find(recovery_info.soid)->second; + const pg_log_entry_t *latest = recovery_state.get_pg_log().get_log().objects.find(recovery_info.soid)->second; if (latest->op == pg_log_entry_t::LOST_REVERT && latest->reverting_to == recovery_info.version) { dout(10) << " got old revert version " << recovery_info.version @@ -446,7 +446,7 @@ void PrimaryLogPG::on_global_recover( bool is_delete) { info.stats.stats.sum.add(stat_diff); - missing_loc.recovered(soid); + recovery_state.object_recovered(soid); publish_stats_to_osd(); dout(10) << "pushed " << soid << " to all replicas" << dendl; map::iterator i = recovering.find(soid); @@ -562,7 +562,7 @@ PerfCounters *PrimaryLogPG::get_logger() bool PrimaryLogPG::is_missing_object(const hobject_t& soid) const { - return pg_log.get_missing().get_items().count(soid); + return recovery_state.get_pg_log().get_missing().get_items().count(soid); } void PrimaryLogPG::maybe_kick_recovery( @@ -570,20 +570,20 @@ void PrimaryLogPG::maybe_kick_recovery( { eversion_t v; bool work_started = false; - if (!missing_loc.needs_recovery(soid, &v)) + if (!recovery_state.get_missing_loc().needs_recovery(soid, &v)) return; map::const_iterator p = recovering.find(soid); if (p != recovering.end()) { dout(7) << "object " << soid << " v " << v << ", already recovering." << dendl; - } else if (missing_loc.is_unfound(soid)) { + } else if (recovery_state.get_missing_loc().is_unfound(soid)) { dout(7) << "object " << soid << " v " << v << ", is unfound." << dendl; } else { dout(7) << "object " << soid << " v " << v << ", recovering." << dendl; PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op(); if (is_missing_object(soid)) { recover_missing(soid, v, cct->_conf->osd_client_op_priority, h); - } else if (missing_loc.is_deleted(soid)) { + } else if (recovery_state.get_missing_loc().is_deleted(soid)) { prep_object_replica_deletes(soid, v, h, &work_started); } else { prep_object_replica_pushes(soid, v, h, &work_started); @@ -609,7 +609,7 @@ bool PrimaryLogPG::is_degraded_or_backfilling_object(const hobject_t& soid) */ if (waiting_for_degraded_object.count(soid)) return true; - if (pg_log.get_missing().get_items().count(soid)) + if (recovery_state.get_pg_log().get_missing().get_items().count(soid)) return true; ceph_assert(!get_acting_recovery_backfill().empty()); for (set::iterator i = get_acting_recovery_backfill().begin(); @@ -738,17 +738,17 @@ void PrimaryLogPG::maybe_force_recovery() PG_STATE_BACKFILL_TOOFULL)) return; - if (pg_log.get_log().approx_size() < + if (recovery_state.get_pg_log().get_log().approx_size() < cct->_conf->osd_max_pg_log_entries * cct->_conf->osd_force_recovery_pg_log_entries_factor) return; // find the oldest missing object - version_t min_version = pg_log.get_log().head.version; + version_t min_version = recovery_state.get_pg_log().get_log().head.version; hobject_t soid; - if (!pg_log.get_missing().get_rmissing().empty()) { - min_version = pg_log.get_missing().get_rmissing().begin()->first; - soid = pg_log.get_missing().get_rmissing().begin()->second; + if (!recovery_state.get_pg_log().get_missing().get_rmissing().empty()) { + min_version = recovery_state.get_pg_log().get_missing().get_rmissing().begin()->first; + soid = recovery_state.get_pg_log().get_missing().get_rmissing().begin()->second; } ceph_assert(!get_acting_recovery_backfill().empty()); for (set::iterator it = get_acting_recovery_backfill().begin(); @@ -1000,7 +1000,7 @@ int PrimaryLogPG::do_command( return -EROFS; } - uint64_t unfound = missing_loc.num_unfound(); + uint64_t unfound = recovery_state.get_missing_loc().num_unfound(); if (!unfound) { ss << "pg has no unfound objects"; return 0; // make command idempotent @@ -1037,7 +1037,8 @@ int PrimaryLogPG::do_command( offset.dump(f.get()); f->close_section(); } - auto &needs_recovery_map = missing_loc.get_needs_recovery(); + auto &needs_recovery_map = recovery_state.get_missing_loc() + .get_needs_recovery(); f->dump_int("num_missing", needs_recovery_map.size()); f->dump_int("num_unfound", get_num_unfound()); map::const_iterator p = @@ -1045,8 +1046,10 @@ int PrimaryLogPG::do_command( { f->open_array_section("objects"); int32_t num = 0; - for (; p != needs_recovery_map.end() && num < cct->_conf->osd_command_max_records; ++p) { - if (missing_loc.is_unfound(p->first)) { + for (; p != needs_recovery_map.end() && + num < cct->_conf->osd_command_max_records; + ++p) { + if (recovery_state.get_missing_loc().is_unfound(p->first)) { f->open_object_section("object"); { f->open_object_section("oid"); @@ -1056,11 +1059,10 @@ int PrimaryLogPG::do_command( p->second.dump(f.get()); // have, need keys { f->open_array_section("locations"); - for (set::iterator r = - missing_loc.get_locations(p->first).begin(); - r != missing_loc.get_locations(p->first).end(); - ++r) - f->dump_stream("shard") << *r; + for (auto &&r : recovery_state.get_missing_loc().get_locations( + p->first)) { + f->dump_stream("shard") << r; + } f->close_section(); } f->close_section(); @@ -1182,12 +1184,12 @@ void PrimaryLogPG::do_pg_op(OpRequestRef op) } map::const_iterator missing_iter = - pg_log.get_missing().get_items().lower_bound(current); + recovery_state.get_pg_log().get_missing().get_items().lower_bound(current); vector::iterator ls_iter = sentries.begin(); hobject_t _max = hobject_t::get_max(); while (1) { const hobject_t &mcand = - missing_iter == pg_log.get_missing().get_items().end() ? + missing_iter == recovery_state.get_pg_log().get_missing().get_items().end() ? _max : missing_iter->first; const hobject_t &lcand = @@ -1232,7 +1234,7 @@ void PrimaryLogPG::do_pg_op(OpRequestRef op) if (candidate.get_namespace() == cct->_conf->osd_hit_set_namespace) continue; - if (missing_loc.is_deleted(candidate)) + if (recovery_state.get_missing_loc().is_deleted(candidate)) continue; // skip wrong namespace @@ -1257,7 +1259,7 @@ void PrimaryLogPG::do_pg_op(OpRequestRef op) } if (next.is_max() && - missing_iter == pg_log.get_missing().get_items().end() && + missing_iter == recovery_state.get_pg_log().get_missing().get_items().end() && ls_iter == sentries.end()) { result = 1; @@ -1338,15 +1340,15 @@ void PrimaryLogPG::do_pg_op(OpRequestRef op) break; } - ceph_assert(snapid == CEPH_NOSNAP || pg_log.get_missing().get_items().empty()); + ceph_assert(snapid == CEPH_NOSNAP || recovery_state.get_pg_log().get_missing().get_items().empty()); map::const_iterator missing_iter = - pg_log.get_missing().get_items().lower_bound(current); + recovery_state.get_pg_log().get_missing().get_items().lower_bound(current); vector::iterator ls_iter = sentries.begin(); hobject_t _max = hobject_t::get_max(); while (1) { const hobject_t &mcand = - missing_iter == pg_log.get_missing().get_items().end() ? + missing_iter == recovery_state.get_pg_log().get_missing().get_items().end() ? _max : missing_iter->first; const hobject_t &lcand = @@ -1387,7 +1389,7 @@ void PrimaryLogPG::do_pg_op(OpRequestRef op) if (candidate.get_namespace() != m->get_hobj().nspace) continue; - if (missing_loc.is_deleted(candidate)) + if (recovery_state.get_missing_loc().is_deleted(candidate)) continue; if (filter && !pgls_filter(filter, candidate, filter_out)) @@ -1397,7 +1399,7 @@ void PrimaryLogPG::do_pg_op(OpRequestRef op) candidate.get_key())); } if (next.is_max() && - missing_iter == pg_log.get_missing().get_items().end() && + missing_iter == recovery_state.get_pg_log().get_missing().get_items().end() && ls_iter == sentries.end()) { result = 1; } @@ -1951,7 +1953,8 @@ void PrimaryLogPG::do_op(OpRequestRef& op) } if (can_backoff && (g_conf()->osd_backoff_on_degraded || - (g_conf()->osd_backoff_on_unfound && missing_loc.is_unfound(head)))) { + (g_conf()->osd_backoff_on_unfound && + recovery_state.get_missing_loc().is_unfound(head)))) { add_backoff(session, head, head); maybe_kick_recovery(head); } else { @@ -8690,7 +8693,7 @@ int PrimaryLogPG::do_copy_get(OpContext *ctx, bufferlist::const_iterator& bp, if (cursor.is_complete()) { // include reqids only in the final step. this is a bit fragile // but it works... - pg_log.get_log().get_object_reqids(ctx->obc->obs.oi.soid, 10, + recovery_state.get_pg_log().get_log().get_object_reqids(ctx->obc->obs.oi.soid, 10, &reply_obj.reqids, &reply_obj.reqid_return_codes); dout(20) << " got reqids" << dendl; @@ -8728,7 +8731,7 @@ void PrimaryLogPG::fill_in_copy_get_noent(OpRequestRef& op, hobject_t oid, uint64_t features = m->get_features(); object_copy_data_t reply_obj; - pg_log.get_log().get_object_reqids(oid, 10, &reply_obj.reqids, + recovery_state.get_pg_log().get_log().get_object_reqids(oid, 10, &reply_obj.reqids, &reply_obj.reqid_return_codes); dout(20) << __func__ << " got reqids " << reply_obj.reqids << dendl; encode(reply_obj, osd_op.outdata, features); @@ -9768,7 +9771,7 @@ int PrimaryLogPG::start_flush( hobject_t next = soid; next.snap = *p; ceph_assert(next.snap < soid.snap); - if (pg_log.get_missing().is_missing(next)) { + if (recovery_state.get_pg_log().get_missing().is_missing(next)) { dout(10) << __func__ << " missing clone is " << next << dendl; if (pmissing) *pmissing = next; @@ -10628,10 +10631,10 @@ void PrimaryLogPG::check_blacklisted_obc_watchers(ObjectContextRef obc) void PrimaryLogPG::populate_obc_watchers(ObjectContextRef obc) { ceph_assert(is_active()); - auto it_objects = pg_log.get_log().objects.find(obc->obs.oi.soid); + auto it_objects = recovery_state.get_pg_log().get_log().objects.find(obc->obs.oi.soid); ceph_assert((recovering.count(obc->obs.oi.soid) || !is_missing_object(obc->obs.oi.soid)) || - (it_objects != pg_log.get_log().objects.end() && // or this is a revert... see recover_primary() + (it_objects != recovery_state.get_pg_log().get_log().objects.end() && // or this is a revert... see recover_primary() it_objects->second->op == pg_log_entry_t::LOST_REVERT && it_objects->second->reverting_to == @@ -10752,11 +10755,11 @@ ObjectContextRef PrimaryLogPG::get_object_context( bool can_create, const map *attrs) { - auto it_objects = pg_log.get_log().objects.find(soid); + auto it_objects = recovery_state.get_pg_log().get_log().objects.find(soid); ceph_assert( - attrs || !pg_log.get_missing().is_missing(soid) || + attrs || !recovery_state.get_pg_log().get_missing().is_missing(soid) || // or this is a revert... see recover_primary() - (it_objects != pg_log.get_log().objects.end() && + (it_objects != recovery_state.get_pg_log().get_log().objects.end() && it_objects->second->op == pg_log_entry_t::LOST_REVERT)); ObjectContextRef obc = object_contexts.lookup(soid); @@ -10954,7 +10957,7 @@ int PrimaryLogPG::find_object_context(const hobject_t& oid, << " snapset " << ssc->snapset << " maps to " << oid << dendl; - if (pg_log.get_missing().is_missing(oid)) { + if (recovery_state.get_pg_log().get_missing().is_missing(oid)) { dout(10) << __func__ << " " << oid << " @" << oid.snap << " snapset " << ssc->snapset << " " << oid << " is missing" << dendl; @@ -11018,7 +11021,7 @@ int PrimaryLogPG::find_object_context(const hobject_t& oid, hobject_t soid(oid.oid, oid.get_key(), ssc->snapset.clones[k], oid.get_hash(), info.pgid.pool(), oid.get_namespace()); - if (pg_log.get_missing().is_missing(soid)) { + if (recovery_state.get_pg_log().get_missing().is_missing(soid)) { dout(20) << __func__ << " " << soid << " missing, try again later" << dendl; if (pmissing) @@ -11222,14 +11225,14 @@ int PrimaryLogPG::recover_missing( int priority, PGBackend::RecoveryHandle *h) { - if (missing_loc.is_unfound(soid)) { + if (recovery_state.get_missing_loc().is_unfound(soid)) { dout(7) << __func__ << " " << soid << " v " << v << " but it is unfound" << dendl; return PULL_NONE; } - if (missing_loc.is_deleted(soid)) { + if (recovery_state.get_missing_loc().is_deleted(soid)) { start_recovery_op(soid); ceph_assert(!recovering.count(soid)); recovering.insert(make_pair(soid, ObjectContextRef())); @@ -11271,13 +11274,13 @@ int PrimaryLogPG::recover_missing( if (soid.snap && soid.snap < CEPH_NOSNAP) { // do we have the head? hobject_t head = soid.get_head(); - if (pg_log.get_missing().is_missing(head)) { + if (recovery_state.get_pg_log().get_missing().is_missing(head)) { if (recovering.count(head)) { dout(10) << " missing but already recovering head " << head << dendl; return PULL_NONE; } else { int r = recover_missing( - head, pg_log.get_missing().get_items().find(head)->second.need, priority, + head, recovery_state.get_pg_log().get_missing().get_items().find(head)->second.need, priority, h); if (r != PULL_NONE) return PULL_HEAD; @@ -11450,7 +11453,7 @@ eversion_t PrimaryLogPG::pick_newest_available(const hobject_t& oid) { eversion_t v; pg_missing_item pmi; - bool is_missing = pg_log.get_missing().is_missing(oid, &pmi); + bool is_missing = recovery_state.get_pg_log().get_missing().is_missing(oid, &pmi); ceph_assert(is_missing); v = pmi.have; dout(10) << "pick_newest_available " << oid << " " << v << " on osd." << osd->whoami << " (local)" << dendl; @@ -11579,24 +11582,24 @@ void PrimaryLogPG::mark_all_unfound_lost( list oids; dout(30) << __func__ << ": log before:\n"; - pg_log.get_log().print(*_dout); + recovery_state.get_pg_log().get_log().print(*_dout); *_dout << dendl; mempool::osd_pglog::list log_entries; utime_t mtime = ceph_clock_now(); map::const_iterator m = - missing_loc.get_needs_recovery().begin(); + recovery_state.get_missing_loc().get_needs_recovery().begin(); map::const_iterator mend = - missing_loc.get_needs_recovery().end(); + recovery_state.get_missing_loc().get_needs_recovery().end(); ObcLockManager manager; eversion_t v = get_next_version(); v.epoch = get_osdmap_epoch(); - uint64_t num_unfound = missing_loc.num_unfound(); + uint64_t num_unfound = recovery_state.get_missing_loc().num_unfound(); while (m != mend) { const hobject_t &oid(m->first); - if (!missing_loc.is_unfound(oid)) { + if (!recovery_state.get_missing_loc().is_unfound(oid)) { // We only care about unfound objects ++m; continue; @@ -11672,7 +11675,7 @@ void PrimaryLogPG::mark_all_unfound_lost( // clear old locations - merge_new_log_entries will have // handled rebuilding missing_loc for each of these // objects if we have the RECOVERY_DELETES flag - missing_loc.recovered(oid); + recovery_state.object_recovered(oid); } } @@ -11918,17 +11921,6 @@ void PrimaryLogPG::on_activate_complete() agent_setup(); } -void PrimaryLogPG::plpg_on_new_interval() -{ - dout(20) << __func__ << " checking missing set deletes flag. missing = " << pg_log.get_missing() << dendl; - - if (!pg_log.get_missing().may_include_deletes && - get_osdmap()->test_flag(CEPH_OSDMAP_RECOVERY_DELETES)) { - pg_log.rebuild_missing_set_with_deletes(osd->store, ch, info); - } - ceph_assert(pg_log.get_missing().may_include_deletes == get_osdmap()->test_flag(CEPH_OSDMAP_RECOVERY_DELETES)); -} - void PrimaryLogPG::on_change(ObjectStore::Transaction *t) { dout(10) << __func__ << dendl; @@ -12070,7 +12062,6 @@ void PrimaryLogPG::plpg_on_pool_change() // clear state. called on recovery completion AND cancellation. void PrimaryLogPG::_clear_recovery_state() { - missing_loc.clear(); #ifdef DEBUG_RECOVERY_OIDS recovering_oids.clear(); #endif @@ -12120,7 +12111,7 @@ void PrimaryLogPG::cancel_pull(const hobject_t &soid) waiting_for_unreadable_object.erase(soid); } if (is_missing_object(soid)) - pg_log.set_last_requested(0); // get recover_primary to start over + recovery_state.set_last_requested(0); finish_degraded_object(soid); } @@ -12153,7 +12144,7 @@ bool PrimaryLogPG::start_recovery_ops( return have_unfound(); } - const auto &missing = pg_log.get_missing(); + const auto &missing = recovery_state.get_pg_log().get_missing(); uint64_t num_unfound = get_num_unfound(); @@ -12221,10 +12212,10 @@ bool PrimaryLogPG::start_recovery_ops( ceph_assert(recovery_ops_active == 0); dout(10) << __func__ << " needs_recovery: " - << missing_loc.get_needs_recovery() + << recovery_state.get_missing_loc().get_needs_recovery() << dendl; dout(10) << __func__ << " missing_loc: " - << missing_loc.get_missing_locs() + << recovery_state.get_missing_loc().get_missing_locs() << dendl; int unfound = get_num_unfound(); if (unfound) { @@ -12294,7 +12285,7 @@ uint64_t PrimaryLogPG::recover_primary(uint64_t max, ThreadPool::TPHandle &handl { ceph_assert(is_primary()); - const auto &missing = pg_log.get_missing(); + const auto &missing = recovery_state.get_pg_log().get_missing(); dout(10) << __func__ << " recovering " << recovering.size() << " in pg," @@ -12309,14 +12300,14 @@ uint64_t PrimaryLogPG::recover_primary(uint64_t max, ThreadPool::TPHandle &handl PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op(); map::const_iterator p = - missing.get_rmissing().lower_bound(pg_log.get_log().last_requested); + missing.get_rmissing().lower_bound(recovery_state.get_pg_log().get_log().last_requested); while (p != missing.get_rmissing().end()) { handle.reset_tp_timeout(); hobject_t soid; version_t v = p->first; - auto it_objects = pg_log.get_log().objects.find(p->second); - if (it_objects != pg_log.get_log().objects.end()) { + auto it_objects = recovery_state.get_pg_log().get_log().objects.find(p->second); + if (it_objects != recovery_state.get_pg_log().get_log().objects.end()) { latest = it_objects->second; ceph_assert(latest->is_update() || latest->is_delete()); soid = latest->soid; @@ -12441,7 +12432,7 @@ uint64_t PrimaryLogPG::recover_primary(uint64_t max, ThreadPool::TPHandle &handl // only advance last_requested if we haven't skipped anything if (!skipped) - pg_log.set_last_requested(v); + recovery_state.set_last_requested(v); } pgbackend->run_recovery_op(h, get_recovery_op_priority()); @@ -12472,7 +12463,8 @@ bool PrimaryLogPG::primary_error( osd->clog->error() << info.pgid << " missing primary copy of " << soid << ", unfound"; else osd->clog->error() << info.pgid << " missing primary copy of " << soid - << ", will try copies on " << missing_loc.get_locations(soid); + << ", will try copies on " + << recovery_state.get_missing_loc().get_locations(soid); return uhoh; } @@ -12615,7 +12607,7 @@ uint64_t PrimaryLogPG::recover_replicas(uint64_t max, ThreadPool::TPHandle &hand handle.reset_tp_timeout(); const hobject_t soid(p->second); - if (missing_loc.is_unfound(soid)) { + if (recovery_state.get_missing_loc().is_unfound(soid)) { dout(10) << __func__ << ": " << soid << " still unfound" << dendl; continue; } @@ -12637,20 +12629,22 @@ uint64_t PrimaryLogPG::recover_replicas(uint64_t max, ThreadPool::TPHandle &hand continue; } - if (missing_loc.is_deleted(soid)) { + if (recovery_state.get_missing_loc().is_deleted(soid)) { dout(10) << __func__ << ": " << soid << " is a delete, removing" << dendl; map::const_iterator r = m.get_items().find(soid); started += prep_object_replica_deletes(soid, r->second.need, h, work_started); continue; } - if (soid.is_snap() && pg_log.get_missing().is_missing(soid.get_head())) { + if (soid.is_snap() && + recovery_state.get_pg_log().get_missing().is_missing( + soid.get_head())) { dout(10) << __func__ << ": " << soid.get_head() << " still missing on primary" << dendl; continue; } - if (pg_log.get_missing().is_missing(soid)) { + if (recovery_state.get_pg_log().get_missing().is_missing(soid)) { dout(10) << __func__ << ": " << soid << " still missing on primary" << dendl; continue; } @@ -13129,7 +13123,7 @@ void PrimaryLogPG::update_range( dout(10) << __func__<< ": bi is current " << dendl; ceph_assert(bi->version == projected_last_update); } else if (bi->version >= info.log_tail) { - if (pg_log.get_log().empty() && projected_log.empty()) { + if (recovery_state.get_pg_log().get_log().empty() && projected_log.empty()) { /* Because we don't move log_tail on split, the log might be * empty even if log_tail != last_update. However, the only * way to get here with an empty log is if log_tail is actually @@ -13165,7 +13159,7 @@ void PrimaryLogPG::update_range( } }; dout(10) << "scanning pg log first" << dendl; - pg_log.get_log().scan_log_after(bi->version, func); + recovery_state.get_pg_log().get_log().scan_log_after(bi->version, func); dout(10) << "scanning projected log" << dendl; projected_log.scan_log_after(bi->version, func); bi->version = projected_last_update; @@ -13225,15 +13219,17 @@ void PrimaryLogPG::check_local() { dout(10) << __func__ << dendl; - ceph_assert(info.last_update >= pg_log.get_tail()); // otherwise we need some help! + ceph_assert( + info.last_update >= + recovery_state.get_pg_log().get_tail()); // otherwise we need some help! if (!cct->_conf->osd_debug_verify_stray_on_activate) return; // just scan the log. set did; - for (list::const_reverse_iterator p = pg_log.get_log().log.rbegin(); - p != pg_log.get_log().log.rend(); + for (list::const_reverse_iterator p = recovery_state.get_pg_log().get_log().log.rbegin(); + p != recovery_state.get_pg_log().get_log().log.rend(); ++p) { if (did.count(p->soid)) continue; @@ -13428,10 +13424,11 @@ bool PrimaryLogPG::hit_set_apply_log() } dout(20) << __func__ << " " << to << " .. " << info.last_update << dendl; - list::const_reverse_iterator p = pg_log.get_log().log.rbegin(); - while (p != pg_log.get_log().log.rend() && p->version > to) + list::const_reverse_iterator p = + recovery_state.get_pg_log().get_log().log.rbegin(); + while (p != recovery_state.get_pg_log().get_log().log.rend() && p->version > to) ++p; - while (p != pg_log.get_log().log.rend() && p->version > from) { + while (p != recovery_state.get_pg_log().get_log().log.rend() && p->version > from) { hit_set->insert(p->soid); ++p; } diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h index a2a7ccb9084aa..4b79394f05885 100644 --- a/src/osd/PrimaryLogPG.h +++ b/src/osd/PrimaryLogPG.h @@ -353,7 +353,7 @@ public: const map> &get_missing_loc_shards() const override { - return missing_loc.get_missing_locs(); + return recovery_state.get_missing_loc().get_missing_locs(); } const map &get_shard_missing() const override { return recovery_state.get_peer_missing(); @@ -364,13 +364,13 @@ public: } using PGBackend::Listener::get_shard_info; const pg_missing_tracker_t &get_local_missing() const override { - return pg_log.get_missing(); + return recovery_state.get_pg_log().get_missing(); } const PGLog &get_log() const override { - return pg_log; + return recovery_state.get_pg_log(); } void add_local_next_event(const pg_log_entry_t& e) override { - pg_log.missing_add_next_entry(e); + recovery_state.add_local_next_event(e); } bool pgb_is_primary() const override { return is_primary(); @@ -1829,7 +1829,8 @@ public: bool is_missing_object(const hobject_t& oid) const; bool is_unreadable_object(const hobject_t &oid) const { return is_missing_object(oid) || - !missing_loc.readable_with_acting(oid, get_actingset()); + !recovery_state.get_missing_loc().readable_with_acting( + oid, get_actingset()); } void maybe_kick_recovery(const hobject_t &soid); void wait_for_unreadable_object(const hobject_t& oid, OpRequestRef op); @@ -1879,7 +1880,6 @@ public: void plpg_on_role_change() override; void plpg_on_pool_change() override; - void plpg_on_new_interval() override; void clear_async_reads(); void on_change(ObjectStore::Transaction *t) override; void on_activate_complete() override; -- 2.39.5