From 47a4a58634d6d5f5a95a374847303c9bf652ab6c Mon Sep 17 00:00:00 2001 From: David Zafman Date: Mon, 23 Jul 2018 16:02:25 -0700 Subject: [PATCH] osd: Fix pg repair to handle object info data_digest mismatched, but replicas match each other. Fixes: https://tracker.ceph.com/issues/25085 Signed-off-by: David Zafman (cherry picked from commit 61e995b2c1b866768151a405641463de067c210c) --- src/common/scrub_types.h | 6 +++++ src/osd/PGBackend.cc | 48 +++++++++++++++++++++++++++++++++++++--- src/osd/PGBackend.h | 3 ++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/common/scrub_types.h b/src/common/scrub_types.h index 88c9a4340c809..190085275d162 100644 --- a/src/common/scrub_types.h +++ b/src/common/scrub_types.h @@ -89,6 +89,12 @@ public: void set_hinfo_corrupted() { errors |= err_t::HINFO_CORRUPTED; } + bool only_data_digest_mismatch_info() const { + return errors == err_t::DATA_DIGEST_MISMATCH_INFO; + } + void clear_data_digest_mismatch_info() { + errors &= ~err_t::DATA_DIGEST_MISMATCH_INFO; + } void encode(bufferlist& bl) const; void decode(bufferlist::iterator& bp); }; diff --git a/src/osd/PGBackend.cc b/src/osd/PGBackend.cc index 8a96348880b49..b5e6094358248 100644 --- a/src/osd/PGBackend.cc +++ b/src/osd/PGBackend.cc @@ -756,7 +756,8 @@ map::const_iterator const map &maps, object_info_t *auth_oi, map &shard_map, - inconsistent_obj_wrapper &object_error) + inconsistent_obj_wrapper &object_error, + bool &digest_match) { eversion_t auth_version; bufferlist first_oi_bl, first_ss_bl, first_hk_bl; @@ -774,6 +775,7 @@ map::const_iterator shards.push_front(get_parent()->whoami_shard()); map::const_iterator auth = maps.end(); + digest_match = true; for (auto &l : shards) { map::const_iterator j = maps.find(l); map::iterator i = @@ -897,6 +899,17 @@ map::const_iterator error_string += " obj_size_info_mismatch"; } + // digest_match will only be true if computed digests are the same + if (auth_version != eversion_t() + && auth->second->objects[obj].digest_present + && i->second.digest_present + && auth->second->objects[obj].digest != i->second.digest) { + digest_match = false; + dout(10) << __func__ << " digest_match = false, " << obj << " data_digest 0x" << std::hex << i->second.digest + << " != data_digest 0x" << auth->second->objects[obj].digest << std::dec + << dendl; + } + // Don't use this particular shard due to previous errors // XXX: For now we can't pick one shard for repair and another's object info or snapset if (shard_info.errors) @@ -952,8 +965,10 @@ void PGBackend::be_compare_scrubmaps( inconsistent_obj_wrapper object_error{*k}; + bool digest_match; map::const_iterator auth = - be_select_auth_object(*k, maps, &auth_oi, shard_map, object_error); + be_select_auth_object(*k, maps, &auth_oi, shard_map, object_error, + digest_match); list auth_list; set object_errors; @@ -974,6 +989,7 @@ void PGBackend::be_compare_scrubmaps( ScrubMap::object& auth_object = auth->second->objects[*k]; set cur_missing; set cur_inconsistent; + bool fix_digest = false; for (auto j = maps.cbegin(); j != maps.cend(); ++j) { if (j == auth) @@ -989,6 +1005,22 @@ void PGBackend::be_compare_scrubmaps( shard_map[j->first], object_error, ss); + + dout(20) << __func__ << (repair ? " repair " : " ") << (parent->get_pool().is_replicated() ? "replicated " : "") + << (j == auth ? "auth" : "") << "shards " << shard_map.size() << (digest_match ? " digest_match " : " ") + << (shard_map[j->first].only_data_digest_mismatch_info() ? "'info mismatch info'" : "") + << dendl; + // If all replicas match, but they don't match object_info we can + // repair it by using missing_digest mechanism + if (repair && parent->get_pool().is_replicated() && j == auth && shard_map.size() > 1 + && digest_match && shard_map[j->first].only_data_digest_mismatch_info() + && auth_object.digest_present) { + // Set in missing_digests + fix_digest = true; + // Clear the error + shard_map[j->first].clear_data_digest_mismatch_info(); + errorstream << pgid << " : soid " << *k << " repairing object info data_digest" << "\n"; + } // Some errors might have already been set in be_select_auth_object() if (shard_map[j->first].errors != 0) { cur_inconsistent.insert(j->first); @@ -1050,9 +1082,19 @@ void PGBackend::be_compare_scrubmaps( if (!cur_inconsistent.empty()) { inconsistent[*k] = cur_inconsistent; } + + if (fix_digest) { + boost::optional data_digest, omap_digest; + assert(auth_object.digest_present); + data_digest = auth_object.digest; + if (auth_object.omap_digest_present) { + omap_digest = auth_object.omap_digest; + } + missing_digest[*k] = make_pair(data_digest, omap_digest); + } if (!cur_inconsistent.empty() || !cur_missing.empty()) { authoritative[*k] = auth_list; - } else if (parent->get_pool().is_replicated()) { + } else if (!fix_digest && parent->get_pool().is_replicated()) { enum { NO = 0, MAYBE = 1, diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h index 51c35a8118027..5a870be8b46eb 100644 --- a/src/osd/PGBackend.h +++ b/src/osd/PGBackend.h @@ -575,7 +575,8 @@ typedef ceph::shared_ptr OSDMapRef; const map &maps, object_info_t *auth_oi, map &shard_map, - inconsistent_obj_wrapper &object_error); + inconsistent_obj_wrapper &object_error, + bool &digest_match); void be_compare_scrubmaps( const map &maps, const set &master_set, -- 2.39.5