From 91d1487ecfb497905a20556baf699ce8dde1cc30 Mon Sep 17 00:00:00 2001 From: David Zafman Date: Sat, 18 Nov 2017 10:16:53 -0800 Subject: [PATCH] osd: Improve the way insufficient targets is handled to be compatible with EC Signed-off-by: David Zafman --- src/osd/PG.cc | 78 ++++++++++++++++++++++++++++++-------------- src/osd/osd_types.cc | 5 ++- src/osd/osd_types.h | 2 ++ 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 03d068efaa5..ac1bfe6a987 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -2679,17 +2679,13 @@ void PG::_update_calc_stats() info.stats.ondisk_log_start = pg_log.get_tail(); info.stats.snaptrimq_len = snap_trimq.size(); - // If actingset is larger then upset we will have misplaced, - // so we will report based on actingset size. + unsigned num_shards = get_osdmap()->get_pg_size(info.pgid.pgid); - // If upset is larger then we will have degraded, - // so we will report based on upset size. - - // If target is the largest of them all, it will contribute to - // the degraded count because num_object_copies is - // computed using target and eventual used to get degraded total. - - unsigned target = get_osdmap()->get_pg_size(info.pgid.pgid); + // In rare case that upset is too large (usually transient), use as target + // for calculations below. + unsigned target = std::max(num_shards, (unsigned)upset.size()); + // Not sure this could ever happen, that actingset > upset + // which only matters if actingset > num_shards. unsigned nrep = std::max(actingset.size(), upset.size()); // calc num_object_copies info.stats.stats.calc_copies(std::max(target, nrep)); @@ -2751,10 +2747,31 @@ void PG::_update_calc_stats() peer.second.stats.stats.sum.num_objects_missing = missing; } + if (pool.info.is_replicated()) { + // Add to missing_target_objects up to target elements (num_objects missing) + assert(target >= missing_target_objects.size()); + unsigned needed = target - missing_target_objects.size(); + for (; needed; --needed) + missing_target_objects.insert(make_pair(num_objects, pg_shard_t(pg_shard_t::NO_OSD))); + } else { + for (unsigned i = 0 ; i < num_shards; ++i) { + shard_id_t shard(i); + bool found = false; + for (const auto& t : missing_target_objects) { + if (std::get<1>(t).shard == shard) { + found = true; + break; + } + } + if (!found) + missing_target_objects.insert(make_pair(num_objects, pg_shard_t(pg_shard_t::NO_OSD,shard))); + } + } + for (const auto& item : missing_target_objects) - dout(20) << __func__ << " missing shard " << item.second << " missing= " << item.first << dendl; + dout(20) << __func__ << " missing shard " << std::get<1>(item) << " missing= " << std::get<0>(item) << dendl; for (const auto& item : acting_source_objects) - dout(20) << __func__ << " acting shard " << item.second << " missing=" << item.first << dendl; + dout(20) << __func__ << " acting shard " << std::get<1>(item) << " missing= " << std::get<0>(item) << dendl; // A misplaced object is not stored on the correct OSD int64_t misplaced = 0; @@ -2766,28 +2783,41 @@ void PG::_update_calc_stats() int64_t extra_missing = -1; - if (!acting_source_objects.empty()) { - auto extra_copy = acting_source_objects.begin(); - extra_missing = extra_copy->first; - acting_source_objects.erase(*extra_copy); + if (pool.info.is_replicated()) { + if (!acting_source_objects.empty()) { + auto extra_copy = acting_source_objects.begin(); + extra_missing = std::get<0>(*extra_copy); + acting_source_objects.erase(extra_copy); + } + } else { // Erasure coded + // Use corresponding shard + for (const auto& a : acting_source_objects) { + if (std::get<1>(a).shard == std::get<1>(*m).shard) { + extra_missing = std::get<0>(a); + acting_source_objects.erase(a); + break; + } + } } - if (extra_missing >= 0 && m->first >= extra_missing) { + if (extra_missing >= 0 && std::get<0>(*m) >= extra_missing) { // We don't know which of the objects on the target // are part of extra_missing so assume are all degraded. - misplaced += m->first - extra_missing; + misplaced += std::get<0>(*m) - extra_missing; degraded += extra_missing; } else { // 1. extra_missing == -1, more targets than sources so degraded - // 2. extra_missing > m->first, so that we know that some extra_missing + // 2. extra_missing > std::get<0>(m), so that we know that some extra_missing // previously degraded are now present on the target. - degraded += m->first; + degraded += std::get<0>(*m); } } - if (target > upset.size()) { - int64_t undersize_degraded = (num_objects * (target - upset.size())); - degraded += undersize_degraded; - dout(20) << __func__ << " undersize degraded " << undersize_degraded << dendl; + // If there are still acting that haven't been accounted for + // then they are misplaced + for (const auto& a : acting_source_objects) { + int64_t extra_misplaced = std::max((int64_t)0, num_objects - std::get<0>(a)); + dout(20) << __func__ << " extra acting misplaced " << extra_misplaced << dendl; + misplaced += extra_misplaced; } dout(20) << __func__ << " degraded " << degraded << dendl; dout(20) << __func__ << " misplaced " << misplaced << dendl; diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index c7246c498d5..d95412e55f6 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -157,8 +157,8 @@ ostream &operator<<(ostream &lhs, const pg_shard_t &rhs) if (rhs.is_undefined()) return lhs << "?"; if (rhs.shard == shard_id_t::NO_SHARD) - return lhs << rhs.osd; - return lhs << rhs.osd << '(' << (unsigned)(rhs.shard) << ')'; + return lhs << rhs.get_osd(); + return lhs << rhs.get_osd() << '(' << (unsigned)(rhs.shard) << ')'; } // -- osd_reqid_t -- @@ -5982,4 +5982,3 @@ void OSDOp::clear_data(vector& ops) } } } - diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 2eb347eabaa..e008a79a9f8 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -138,6 +138,7 @@ WRITE_CLASS_DENC(osd_reqid_t) struct pg_shard_t { + static const int32_t NO_OSD = 0x7fffffff; int32_t osd; shard_id_t shard; pg_shard_t() : osd(-1), shard(shard_id_t::NO_SHARD) {} @@ -146,6 +147,7 @@ struct pg_shard_t { bool is_undefined() const { return osd == -1; } + string get_osd() const { return (osd == NO_OSD ? "NONE" : to_string(osd)); } void encode(bufferlist &bl) const; void decode(bufferlist::iterator &bl); void dump(Formatter *f) const { -- 2.39.5