From 9e512985901c7d760d2c3c60883201979517bf95 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Fri, 5 Apr 2019 17:24:23 -0700 Subject: [PATCH] osd/: move last_..._to_applied, backfill_targets, async_recovery_targets to PeeringState Signed-off-by: Samuel Just --- src/osd/PG.cc | 14 ++----- src/osd/PG.h | 7 ---- src/osd/PeeringState.cc | 13 +++++- src/osd/PeeringState.h | 16 +++++++- src/osd/PrimaryLogPG.cc | 90 +++++++++++++++++++++-------------------- src/osd/PrimaryLogPG.h | 14 ++++++- 6 files changed, 90 insertions(+), 64 deletions(-) diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 26d5693802ceb..5358a4746d830 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -196,7 +196,6 @@ PG::PG(OSDService *o, OSDMapRef curmap, last_update_ondisk(recovery_state.last_update_ondisk), last_complete_ondisk(recovery_state.last_complete_ondisk), last_update_applied(recovery_state.last_update_applied), - last_rollback_info_trimmed_to_applied(recovery_state.last_rollback_info_trimmed_to_applied), peer_info(recovery_state.peer_info), peer_missing(recovery_state.peer_missing), peer_log_requested(recovery_state.peer_log_requested), @@ -204,8 +203,6 @@ PG::PG(OSDService *o, OSDMapRef curmap, peer_last_complete_ondisk(recovery_state.peer_last_complete_ondisk), min_last_complete_ondisk(recovery_state.min_last_complete_ondisk), pg_trim_to(recovery_state.pg_trim_to), - backfill_targets(recovery_state.backfill_targets), - async_recovery_targets(recovery_state.async_recovery_targets), might_have_unfound(recovery_state.might_have_unfound), missing_loc(recovery_state.missing_loc), pg_id(p), @@ -746,7 +743,6 @@ void PG::clear_recovery_state() { dout(10) << "clear_recovery_state" << dendl; - pg_log.reset_recovery_pointers(); finish_sync_event = 0; hobject_t soid; @@ -757,8 +753,6 @@ void PG::clear_recovery_state() finish_recovery_op(soid, true); } - async_recovery_targets.clear(); - backfill_targets.clear(); backfill_info.clear(); peer_backfill_info.clear(); waiting_on_backfill.clear(); @@ -2116,7 +2110,7 @@ void PG::clear_scrub_reserved() void PG::scrub_reserve_replicas() { - ceph_assert(backfill_targets.empty()); + ceph_assert(recovery_state.get_backfill_targets().empty()); for (set::iterator i = acting_recovery_backfill.begin(); i != acting_recovery_backfill.end(); ++i) { @@ -2133,7 +2127,7 @@ void PG::scrub_reserve_replicas() void PG::scrub_unreserve_replicas() { - ceph_assert(backfill_targets.empty()); + ceph_assert(recovery_state.get_backfill_targets().empty()); for (set::iterator i = acting_recovery_backfill.begin(); i != acting_recovery_backfill.end(); ++i) { @@ -2151,7 +2145,7 @@ void PG::scrub_unreserve_replicas() void PG::_scan_rollback_obs(const vector &rollback_obs) { ObjectStore::Transaction t; - eversion_t trimmed_to = last_rollback_info_trimmed_to_applied; + eversion_t trimmed_to = recovery_state.get_last_rollback_info_trimmed_to_applied(); for (vector::const_iterator i = rollback_obs.begin(); i != rollback_obs.end(); ++i) { @@ -2567,7 +2561,7 @@ void PG::scrub(epoch_t queued, ThreadPool::TPHandle &handle) } if (!scrubber.active) { - ceph_assert(backfill_targets.empty()); + ceph_assert(recovery_state.get_backfill_targets().empty()); scrubber.deep = state_test(PG_STATE_DEEP_SCRUB); diff --git a/src/osd/PG.h b/src/osd/PG.h index ab58099816435..2c20d80c18a55 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -193,7 +193,6 @@ protected: eversion_t &last_update_ondisk; eversion_t &last_complete_ondisk; eversion_t &last_update_applied; - eversion_t &last_rollback_info_trimmed_to_applied; map &peer_info; map &peer_missing; set &peer_log_requested; @@ -201,8 +200,6 @@ protected: map &peer_last_complete_ondisk; eversion_t &min_last_complete_ondisk; eversion_t &pg_trim_to; - set &backfill_targets; - set &async_recovery_targets; set &might_have_unfound; MissingLoc &missing_loc; @@ -807,10 +804,6 @@ protected: std::atomic local_num_bytes = 0; public: - bool is_backfill_targets(pg_shard_t osd) { - return recovery_state.is_backfill_targets(osd); - } - // Space reserved for backfill is primary_num_bytes - local_num_bytes // Don't care that difference itself isn't atomic uint64_t get_reserved_num_bytes() { diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index 73868cdd4fcf1..76bb83110124e 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -769,6 +769,12 @@ void PeeringState::init_primary_up_acting( ceph_assert(primary.osd == new_acting_primary); } +void PeeringState::clear_recovery_state() +{ + async_recovery_targets.clear(); + backfill_targets.clear(); +} + void PeeringState::clear_primary_state() { psdout(10) << "clear_primary_state" << dendl; @@ -788,6 +794,9 @@ void PeeringState::clear_primary_state() need_up_thru = false; missing_loc.clear(); pg_log.reset_recovery_pointers(); + + clear_recovery_state(); + pl->clear_primary_state(); } @@ -2623,6 +2632,7 @@ void PeeringState::try_mark_clean() share_pg_info(); pl->publish_stats_to_osd(); + clear_recovery_state(); } void PeeringState::split_into( @@ -3001,7 +3011,7 @@ void PeeringState::update_calc_stats() int64_t peer_num_objects = peer.second.stats.stats.sum.num_objects; // Backfill targets always track num_objects accurately // all other peers track missing accurately. - if (is_backfill_targets(peer.first)) { + if (is_backfill_target(peer.first)) { missing = std::max((int64_t)0, num_objects - peer_num_objects); } else { if (peer_missing.count(peer.first)) { @@ -4917,6 +4927,7 @@ PeeringState::Clean::Clean(my_context ctx) ceph_abort(); } + ps->try_mark_clean(); context< PeeringMachine >().get_cur_transaction()->register_on_commit( diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h index b8f2b0ea58d67..696e10b4d3f6c 100644 --- a/src/osd/PeeringState.h +++ b/src/osd/PeeringState.h @@ -1288,6 +1288,7 @@ public: const vector &newacting, int new_up_primary, int new_acting_primary); + void clear_recovery_state(); void clear_primary_state(); void check_past_interval_bounds() const; bool set_force_recovery(bool b); @@ -1547,6 +1548,10 @@ public: return last_peering_reset; } + eversion_t get_last_rollback_info_trimmed_to_applied() const { + return last_rollback_info_trimmed_to_applied; + } + /// Returns stable reference to internal pool structure const PGPool &get_pool() const { return pool; @@ -1661,9 +1666,18 @@ public: return up_primary.osd; } - bool is_backfill_targets(pg_shard_t osd) { + bool is_backfill_target(pg_shard_t osd) const { return backfill_targets.count(osd); } + const set &get_backfill_targets() const { + return backfill_targets; + } + bool is_async_recovery_target(pg_shard_t peer) const { + return async_recovery_targets.count(peer); + } + const set &get_async_recovery_targets() const { + return async_recovery_targets; + } bool state_test(uint64_t m) const { return (state & m) != 0; } void state_set(uint64_t m) { state |= m; } diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 75ce5e4c37b00..dc866d8d0ac18 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -553,7 +553,7 @@ bool PrimaryLogPG::should_send_op( hoid <= last_backfill_started || hoid <= peer_info[peer].last_backfill; if (!should_send) { - ceph_assert(is_backfill_targets(peer)); + ceph_assert(is_backfill_target(peer)); dout(10) << __func__ << " issue_repop shipping empty opt to osd." << peer << ", object " << hoid << " beyond std::max(last_backfill_started " @@ -561,7 +561,7 @@ bool PrimaryLogPG::should_send_op( << peer_info[peer].last_backfill << ")" << dendl; return should_send; } - if (async_recovery_targets.count(peer) && peer_missing[peer].is_missing(hoid)) { + if (is_async_recovery_target(peer) && peer_missing[peer].is_missing(hoid)) { should_send = false; dout(10) << __func__ << " issue_repop shipping empty opt to osd." << peer << ", object " << hoid @@ -648,14 +648,14 @@ bool PrimaryLogPG::is_degraded_or_backfilling_object(const hobject_t& soid) // This will not block the op and the object is async recovered later. if (peer_missing_entry != peer_missing.end() && peer_missing_entry->second.get_items().count(soid)) { - if (async_recovery_targets.count(peer)) + if (is_async_recovery_target(peer)) continue; else return true; } // Object is degraded if after last_backfill AND // we are backfilling it - if (is_backfill_targets(peer) && + if (is_backfill_target(peer) && peer_info[peer].last_backfill <= soid && last_backfill_started >= soid && backfills_in_flight.count(soid)) @@ -666,7 +666,7 @@ bool PrimaryLogPG::is_degraded_or_backfilling_object(const hobject_t& soid) bool PrimaryLogPG::is_degraded_on_async_recovery_target(const hobject_t& soid) { - for (auto &i: async_recovery_targets) { + for (auto &i: get_async_recovery_targets()) { auto peer_missing_entry = peer_missing.find(i); if (peer_missing_entry != peer_missing.end() && peer_missing_entry->second.get_items().count(soid)) { @@ -1870,8 +1870,8 @@ void PrimaryLogPG::do_request( hobject_t PrimaryLogPG::earliest_backfill() const { hobject_t e = hobject_t::get_max(); - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t bt = *i; map::const_iterator iter = peer_info.find(bt); @@ -4190,7 +4190,7 @@ void PrimaryLogPG::do_scan( pg_shard_t from = m->from; // Check that from is in backfill_targets vector - ceph_assert(is_backfill_targets(from)); + ceph_assert(is_backfill_target(from)); BackfillInterval& bi = peer_backfill_info[from]; bi.begin = m->begin; @@ -4203,7 +4203,9 @@ void PrimaryLogPG::do_scan( if (waiting_on_backfill.erase(from)) { if (waiting_on_backfill.empty()) { - ceph_assert(peer_backfill_info.size() == backfill_targets.size()); + ceph_assert( + peer_backfill_info.size() == + get_backfill_targets().size()); finish_recovery_op(hobject_t::get_max()); } } else { @@ -8541,8 +8543,8 @@ void PrimaryLogPG::apply_stats( info.stats.stats.add(delta_stats); info.stats.stats.floor(0); - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t bt = *i; pg_info_t& pinfo = peer_info[bt]; @@ -10479,8 +10481,8 @@ void PrimaryLogPG::issue_repop(RepGather *repop, OpContext *ctx) } bool requires_missing_loc = false; - for (set::iterator i = async_recovery_targets.begin(); - i != async_recovery_targets.end(); + for (set::iterator i = get_async_recovery_targets().begin(); + i != get_async_recovery_targets().end(); ++i) { if (*i == get_primary() || !peer_missing[*i].is_missing(soid)) continue; requires_missing_loc = true; @@ -12087,14 +12089,14 @@ void PrimaryLogPG::on_activate_complete() publish_stats_to_osd(); - if (!backfill_targets.empty()) { + if (get_backfill_targets().size()) { last_backfill_started = earliest_backfill(); new_backfill = true; ceph_assert(!last_backfill_started.is_max()); - dout(5) << __func__ << ": bft=" << backfill_targets + dout(5) << __func__ << ": bft=" << get_backfill_targets() << " from " << last_backfill_started << dendl; - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { dout(5) << "target shard " << *i << " from " << peer_info[*i].last_backfill @@ -12370,7 +12372,7 @@ bool PrimaryLogPG::start_recovery_ops( bool deferred_backfill = false; if (recovering.empty() && state_test(PG_STATE_BACKFILLING) && - !backfill_targets.empty() && started < max && + !get_backfill_targets().empty() && started < max && missing.num_missing() == 0 && waiting_on_backfill.empty()) { if (get_osdmap()->test_flag(CEPH_OSDMAP_NOBACKFILL)) { @@ -12768,7 +12770,7 @@ uint64_t PrimaryLogPG::recover_replicas(uint64_t max, ThreadPool::TPHandle &hand ceph_assert(pm != peer_missing.end()); auto nm = pm->second.num_missing(); if (nm != 0) { - if (async_recovery_targets.count(p)) { + if (is_async_recovery_target(p)) { async_by_num_missing.push_back(make_pair(nm, p)); } else { replicas_by_num_missing.push_back(make_pair(nm, p)); @@ -12857,8 +12859,8 @@ uint64_t PrimaryLogPG::recover_replicas(uint64_t max, ThreadPool::TPHandle &hand hobject_t PrimaryLogPG::earliest_peer_backfill() const { hobject_t e = hobject_t::get_max(); - for (set::const_iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t peer = *i; map::const_iterator iter = @@ -12875,8 +12877,8 @@ bool PrimaryLogPG::all_peer_done() const // Primary hasn't got any more objects ceph_assert(backfill_info.empty()); - for (set::const_iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t bt = *i; map::const_iterator piter = @@ -12923,11 +12925,11 @@ uint64_t PrimaryLogPG::recover_backfill( ThreadPool::TPHandle &handle, bool *work_started) { dout(10) << __func__ << " (" << max << ")" - << " bft=" << backfill_targets + << " bft=" << get_backfill_targets() << " last_backfill_started " << last_backfill_started << (new_backfill ? " new_backfill":"") << dendl; - ceph_assert(!backfill_targets.empty()); + ceph_assert(!get_backfill_targets().empty()); // Initialize from prior backfill state if (new_backfill) { @@ -12936,8 +12938,8 @@ uint64_t PrimaryLogPG::recover_backfill( new_backfill = false; // initialize BackfillIntervals - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { peer_backfill_info[*i].reset(peer_info[*i].last_backfill); } @@ -12947,8 +12949,8 @@ uint64_t PrimaryLogPG::recover_backfill( pending_backfill_updates.clear(); } - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { dout(10) << "peer osd." << *i << " info " << peer_info[*i] @@ -12966,8 +12968,8 @@ uint64_t PrimaryLogPG::recover_backfill( vector > to_remove; set add_to_stat; - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { peer_backfill_info[*i].trim_to( std::max(peer_info[*i].last_backfill, last_backfill_started)); @@ -12988,8 +12990,8 @@ uint64_t PrimaryLogPG::recover_backfill( dout(20) << " my backfill interval " << backfill_info << dendl; bool sent_scan = false; - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t bt = *i; BackfillInterval& pbi = peer_backfill_info[bt]; @@ -13029,8 +13031,8 @@ uint64_t PrimaryLogPG::recover_backfill( if (check < backfill_info.begin) { set check_targets; - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t bt = *i; BackfillInterval& pbi = peer_backfill_info[bt]; @@ -13062,8 +13064,8 @@ uint64_t PrimaryLogPG::recover_backfill( eversion_t& obj_v = backfill_info.objects.begin()->second; vector need_ver_targs, missing_targs, keep_ver_targs, skip_targs; - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t bt = *i; BackfillInterval& pbi = peer_backfill_info[bt]; @@ -13128,7 +13130,7 @@ uint64_t PrimaryLogPG::recover_backfill( } dout(20) << "need_ver_targs=" << need_ver_targs << " keep_ver_targs=" << keep_ver_targs << dendl; - dout(20) << "backfill_targets=" << backfill_targets + dout(20) << "backfill_targets=" << get_backfill_targets() << " missing_targs=" << missing_targs << " skip_targs=" << skip_targs << dendl; @@ -13204,8 +13206,8 @@ uint64_t PrimaryLogPG::recover_backfill( pending_backfill_updates.erase(i++)) { dout(20) << " pending_backfill_update " << i->first << dendl; ceph_assert(i->first > new_last_backfill); - for (set::iterator j = backfill_targets.begin(); - j != backfill_targets.end(); + for (set::const_iterator j = get_backfill_targets().begin(); + j != get_backfill_targets().end(); ++j) { pg_shard_t bt = *j; pg_info_t& pinfo = peer_info[bt]; @@ -13230,8 +13232,8 @@ uint64_t PrimaryLogPG::recover_backfill( // If new_last_backfill == MAX, then we will send OP_BACKFILL_FINISH to // all the backfill targets. Otherwise, we will move last_backfill up on // those targets need it and send OP_BACKFILL_PROGRESS to them. - for (set::iterator i = backfill_targets.begin(); - i != backfill_targets.end(); + for (set::const_iterator i = get_backfill_targets().begin(); + i != get_backfill_targets().end(); ++i) { pg_shard_t bt = *i; pg_info_t& pinfo = peer_info[bt]; @@ -13669,8 +13671,8 @@ void PrimaryLogPG::hit_set_persist() // look just at that. This is necessary because our transactions // may include a modify of the new hit_set *and* a delete of the // old one, and this may span the backfill boundary. - for (set::iterator p = backfill_targets.begin(); - p != backfill_targets.end(); + for (set::const_iterator p = get_backfill_targets().begin(); + p != get_backfill_targets().end(); ++p) { ceph_assert(peer_info.count(*p)); const pg_info_t& pi = peer_info[*p]; diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h index a596658661b1e..ad5876f7445d1 100644 --- a/src/osd/PrimaryLogPG.h +++ b/src/osd/PrimaryLogPG.h @@ -339,7 +339,7 @@ public: return actingset; } const set &get_backfill_shards() const override { - return backfill_targets; + return get_backfill_targets(); } std::ostream& gen_dbg_prefix(std::ostream& out) const override { @@ -1832,6 +1832,18 @@ public: void wait_for_unreadable_object(const hobject_t& oid, OpRequestRef op); void wait_for_all_missing(OpRequestRef op); + bool is_backfill_target(pg_shard_t osd) const { + return recovery_state.is_backfill_target(osd); + } + const set &get_backfill_targets() const { + return recovery_state.get_backfill_targets(); + } + bool is_async_recovery_target(pg_shard_t peer) const { + return recovery_state.is_async_recovery_target(peer); + } + const set &get_async_recovery_targets() const { + return recovery_state.get_async_recovery_targets(); + } bool is_degraded_or_backfilling_object(const hobject_t& oid); bool is_degraded_on_async_recovery_target(const hobject_t& soid); void wait_for_degraded_object(const hobject_t& oid, OpRequestRef op); -- 2.39.5