From e1075f107372c97580b984c74666027fc2e71daf Mon Sep 17 00:00:00 2001 From: David Zafman Date: Wed, 25 Oct 2017 09:37:00 -0700 Subject: [PATCH] osd: Rewrite _update_calc_stats() to make it cleaner and more accurate Signed-off-by: David Zafman --- src/osd/PG.cc | 160 +++++++++++++++++++++----------------------------- 1 file changed, 66 insertions(+), 94 deletions(-) diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 425ba6377cd..e4aa8e18823 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -1885,12 +1885,15 @@ void PG::activate(ObjectStore::Transaction& t, build_might_have_unfound(); + // XXX: Has _update_calc_stats() run? If so, base on num_objects_degraded? state_set(PG_STATE_DEGRADED); if (have_unfound()) discover_all_missing(query_map); } // degraded? + // num_objects_degraded if calculated should reflect this too, unless no + // missing and we are about to go clean. if (get_osdmap()->get_pg_size(info.pgid.pgid) > actingset.size()) { state_set(PG_STATE_DEGRADED); state_set(PG_STATE_UNDERSIZED); @@ -2689,115 +2692,83 @@ void PG::_update_calc_stats() info.stats.stats.sum.num_objects_degraded = 0; info.stats.stats.sum.num_objects_unfound = 0; info.stats.stats.sum.num_objects_misplaced = 0; - if ((is_degraded() || is_undersized() || !is_clean()) && is_peered()) { - // NOTE: we only generate copies, degraded, misplaced and unfound + if (!is_clean() && is_peered()) { + dout(20) << __func__ << " not clean" << dendl; + + assert(!actingbackfill.empty()); + + // NOTE: we only generate degraded, misplaced and unfound // values for the summation, not individual stat categories. int64_t num_objects = info.stats.stats.sum.num_objects; - // Objects that have arrived backfilled to up OSDs (not in acting) - int64_t backfilled = 0; - // A misplaced object is not stored on the correct OSD - int64_t misplaced = 0; - // Total of object copies/shards found - int64_t object_copies = 0; - // Total number of OSDs with misplaced objects - int num_misplaced = 0; - - // num_objects_missing on each peer - for (map::iterator pi = - peer_info.begin(); - pi != peer_info.end(); - ++pi) { - map::const_iterator pm = - peer_missing.find(pi->first); - if (pm != peer_missing.end()) { - pi->second.stats.stats.sum.num_objects_missing = - pm->second.num_missing(); - } - } + // Objects missing from up nodes, sorted by # objects. + boost::container::flat_set> missing_target_objects; + // Objects missing from nodes not in up, sort by # objects + boost::container::flat_set> acting_source_objects; - // Add recovery objects not part of actingbackfill to be used to reduce - // degraded and account as misplaced. - for (const auto& peer : peer_info) { - if (actingbackfill.find(peer.first) == actingbackfill.end()) { - object_copies += peer.second.stats.stats.sum.num_objects; - misplaced += peer.second.stats.stats.sum.num_objects; - ++num_misplaced; - } - } - if (object_copies) - dout(20) << __func__ << " objects not part of up/acting " << object_copies << dendl; + int64_t missing; - // Objects backfilled, sorted by # objects. - set> backfill_target_objects; + // Primary first + missing = pg_log.get_missing().num_missing(); + assert(actingbackfill.count(pg_whoami)); + if (upset.count(pg_whoami)) { + missing_target_objects.insert(make_pair(missing, pg_whoami)); + } else { + acting_source_objects.insert(make_pair(missing, pg_whoami)); + } + info.stats.stats.sum.num_objects_missing_on_primary = missing; - assert(!actingbackfill.empty()); - for (set::iterator i = actingbackfill.begin(); - i != actingbackfill.end(); - ++i) { - const pg_shard_t &p = *i; - bool in_up = (upset.find(p) != upset.end()); - - // recovery Compute total objects excluding num_missing - // - not in up Compute misplaced objects excluding num_missing - // backfill Compute total objects already backfilled - if (!is_backfill_targets(p)) { - unsigned osd_missing, osd_objects; - // primary handling - if (p == pg_whoami) { - osd_missing = pg_log.get_missing().num_missing(); - info.stats.stats.sum.num_objects_missing_on_primary = - osd_missing; - } else { - assert(peer_missing.count(p)); - osd_missing = peer_missing[p].num_missing(); + // All other peers + for (auto& peer : peer_info) { + // Ignore other peers until we add code to look at detailed missing + // information. (recovery) + if (!actingbackfill.count(peer.first)) { + continue; + } + missing = 0; + // Backfill targets always track num_objects accurately + // all other peers track missing accurately. + if (is_backfill_targets(peer.first)) { + missing = std::max((int64_t)0, num_objects - peer.second.stats.stats.sum.num_objects); + if (missing < 0) missing = 0; + } else { + if (peer_missing.count(peer.first)) { + missing = peer_missing[peer.first].num_missing(); + } else { + dout(20) << __func__ << " no peer_mssing found for " << peer.first << dendl; } - - osd_objects = std::max(0, num_objects - osd_missing); - object_copies += osd_objects; - // Count non-missing objects not in up as misplaced - if (!in_up) { - misplaced += osd_objects; - ++num_misplaced; - } + } + if (upset.count(peer.first)) { + missing_target_objects.insert(make_pair(missing, peer.first)); } else { - // If this peer has more objects then it should, ignore them - int64_t osd_backfilled = std::min(num_objects, - peer_info[p].stats.stats.sum.num_objects); - backfill_target_objects.insert(make_pair(osd_backfilled, p)); - backfilled += osd_backfilled; + acting_source_objects.insert(make_pair(missing, peer.first)); } + peer.second.stats.stats.sum.num_objects_missing = missing; } - // Only include backfill targets below pool size into the object_copies - // count. Use the most-full targets. - int num_backfill_shards_seen = 0; - for (auto i = backfill_target_objects.rbegin(); - i != backfill_target_objects.rend(); - ++i) { - if (actingset.size() + num_backfill_shards_seen < pool.info.size) { - object_copies += i->first; - ++num_backfill_shards_seen; + for (const auto& item : missing_target_objects) + dout(20) << __func__ << " missing shard " << item.second << " missing= " << item.first << dendl; + for (const auto& item : acting_source_objects) + dout(20) << __func__ << " acting shard " << item.second << " missing=" << item.first << dendl; + + // A misplaced object is not stored on the correct OSD + int64_t misplaced = 0; + // a degraded objects has fewer replicas or EC shards than the pool specifies. + int64_t degraded = 0; + + for (auto m = missing_target_objects.rbegin(); + m != missing_target_objects.rend(); ++m) { + + if (!acting_source_objects.empty()) { + auto extra_copy = acting_source_objects.begin(); + misplaced += m->first - extra_copy->first; + degraded += extra_copy->first; + acting_source_objects.erase(*extra_copy); } else { - break; + degraded += m->first; } } - // Any objects that have been backfilled to up OSDs can deducted from misplaced - // adjusted by the assumption that backfill is happening approximately evenly - // across backfill nodes. Use the most-full targets. - int64_t adjust_misplaced = 0; - for (auto i = backfill_target_objects.rbegin(); - i != backfill_target_objects.rend() && num_misplaced; - ++i, --num_misplaced) { - adjust_misplaced += i->first; - } - misplaced = std::max(0, misplaced - adjust_misplaced); - - // a degraded objects has fewer replicas or EC shards than the - // pool specifies. num_object_copies will never be smaller than target * num_objects. - int64_t degraded = std::max(0, info.stats.stats.sum.num_object_copies - object_copies); - info.stats.stats.sum.num_objects_degraded = degraded; info.stats.stats.sum.num_objects_unfound = get_num_unfound(); info.stats.stats.sum.num_objects_misplaced = misplaced; @@ -7398,6 +7369,7 @@ boost::statechart::result PG::RecoveryState::Active::react(const AdvMap& advmap) pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid)) { if (pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid) <= pg->actingset.size()) { pg->state_clear(PG_STATE_UNDERSIZED); + // XXX: Base this on num_objects_degraded instead? if (pg->needs_recovery()) { pg->state_set(PG_STATE_DEGRADED); } else { -- 2.39.5