From 34eb549cd422171540e1dbd3fd04dc44708483f2 Mon Sep 17 00:00:00 2001 From: David Zafman Date: Tue, 28 Jan 2014 19:18:32 -0800 Subject: [PATCH] osd: Move the rest of scrubbing routines to the backend Move enum scrub_error_type to osd_types.h Move PG::_compare_scrub_objects to ReplicatedBackend::be_compare_scrub_objects Move PG::_select_auth_object to ReplicatedBackend::be_select_auth_object Move PG::_compare_scrubmaps to ReplicatedBackend::be_compare_scrubmaps Signed-off-by: David Zafman --- src/osd/PG.cc | 210 +---------------------------------- src/osd/PG.h | 18 --- src/osd/PGBackend.h | 16 +++ src/osd/ReplicatedBackend.cc | 210 +++++++++++++++++++++++++++++++++++ src/osd/ReplicatedBackend.h | 16 +++ src/osd/osd_types.h | 5 + 6 files changed, 251 insertions(+), 224 deletions(-) diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 7ca1d1616a244..4c1d909f00ebd 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -3750,212 +3750,7 @@ bool PG::scrub_gather_replica_maps() } } -enum PG::error_type PG::_compare_scrub_objects(ScrubMap::object &auth, - ScrubMap::object &candidate, - ostream &errorstream) -{ - enum PG::error_type error = CLEAN; - if (candidate.read_error) { - // This can occur on stat() of a shallow scrub, but in that case size will - // be invalid, and this will be over-ridden below. - error = DEEP_ERROR; - errorstream << "candidate had a read error"; - } - if (auth.digest_present && candidate.digest_present) { - if (auth.digest != candidate.digest) { - if (error != CLEAN) - errorstream << ", "; - error = DEEP_ERROR; - - errorstream << "digest " << candidate.digest - << " != known digest " << auth.digest; - } - } - if (auth.omap_digest_present && candidate.omap_digest_present) { - if (auth.omap_digest != candidate.omap_digest) { - if (error != CLEAN) - errorstream << ", "; - error = DEEP_ERROR; - - errorstream << "omap_digest " << candidate.omap_digest - << " != known omap_digest " << auth.omap_digest; - } - } - // Shallow error takes precendence because this will be seen by - // both types of scrubs. - if (auth.size != candidate.size) { - if (error != CLEAN) - errorstream << ", "; - error = SHALLOW_ERROR; - errorstream << "size " << candidate.size - << " != known size " << auth.size; - } - for (map::const_iterator i = auth.attrs.begin(); - i != auth.attrs.end(); - ++i) { - if (!candidate.attrs.count(i->first)) { - if (error != CLEAN) - errorstream << ", "; - error = SHALLOW_ERROR; - errorstream << "missing attr " << i->first; - } else if (candidate.attrs.find(i->first)->second.cmp(i->second)) { - if (error != CLEAN) - errorstream << ", "; - error = SHALLOW_ERROR; - errorstream << "attr value mismatch " << i->first; - } - } - for (map::const_iterator i = candidate.attrs.begin(); - i != candidate.attrs.end(); - ++i) { - if (!auth.attrs.count(i->first)) { - if (error != CLEAN) - errorstream << ", "; - error = SHALLOW_ERROR; - errorstream << "extra attr " << i->first; - } - } - return error; -} - - - -map::const_iterator PG::_select_auth_object( - const hobject_t &obj, - const map &maps) -{ - map::const_iterator auth = maps.end(); - for (map::const_iterator j = maps.begin(); - j != maps.end(); - ++j) { - map::iterator i = - j->second->objects.find(obj); - if (i == j->second->objects.end()) { - continue; - } - if (auth == maps.end()) { - // Something is better than nothing - // TODO: something is NOT better than nothing, do something like - // unfound_lost if no valid copies can be found, or just mark unfound - auth = j; - dout(10) << __func__ << ": selecting osd " << j->first - << " for obj " << obj - << ", auth == maps.end()" - << dendl; - continue; - } - if (i->second.read_error) { - // scrub encountered read error, probably corrupt - dout(10) << __func__ << ": rejecting osd " << j->first - << " for obj " << obj - << ", read_error" - << dendl; - continue; - } - map::iterator k = i->second.attrs.find(OI_ATTR); - if (k == i->second.attrs.end()) { - // no object info on object, probably corrupt - dout(10) << __func__ << ": rejecting osd " << j->first - << " for obj " << obj - << ", no oi attr" - << dendl; - continue; - } - bufferlist bl; - bl.push_back(k->second); - object_info_t oi; - try { - bufferlist::iterator bliter = bl.begin(); - ::decode(oi, bliter); - } catch (...) { - dout(10) << __func__ << ": rejecting osd " << j->first - << " for obj " << obj - << ", corrupt oi attr" - << dendl; - // invalid object info, probably corrupt - continue; - } - if (oi.size != i->second.size) { - // invalid size, probably corrupt - dout(10) << __func__ << ": rejecting osd " << j->first - << " for obj " << obj - << ", size mismatch" - << dendl; - // invalid object info, probably corrupt - continue; - } - dout(10) << __func__ << ": selecting osd " << j->first - << " for obj " << obj - << dendl; - auth = j; - } - return auth; -} - -void PG::_compare_scrubmaps(const map &maps, - map > &missing, - map > &inconsistent, - map &authoritative, - map > &invalid_snapcolls, - ostream &errorstream) -{ - map::const_iterator i; - map::const_iterator j; - set master_set; - - // Construct master set - for (j = maps.begin(); j != maps.end(); ++j) { - for (i = j->second->objects.begin(); i != j->second->objects.end(); ++i) { - master_set.insert(i->first); - } - } - // Check maps against master set and each other - for (set::const_iterator k = master_set.begin(); - k != master_set.end(); - ++k) { - map::const_iterator auth = _select_auth_object(*k, maps); - assert(auth != maps.end()); - set cur_missing; - set cur_inconsistent; - for (j = maps.begin(); j != maps.end(); ++j) { - if (j == auth) - continue; - if (j->second->objects.count(*k)) { - // Compare - stringstream ss; - enum PG::error_type error = _compare_scrub_objects(auth->second->objects[*k], - j->second->objects[*k], - ss); - if (error != CLEAN) { - cur_inconsistent.insert(j->first); - if (error == SHALLOW_ERROR) - ++scrubber.shallow_errors; - else - ++scrubber.deep_errors; - errorstream << info.pgid << " osd." << acting[j->first] - << ": soid " << *k << " " << ss.str() << std::endl; - } - } else { - cur_missing.insert(j->first); - ++scrubber.shallow_errors; - errorstream << info.pgid - << " osd." << acting[j->first] - << " missing " << *k << std::endl; - } - } - assert(auth != maps.end()); - if (!cur_missing.empty()) { - missing[*k] = cur_missing; - } - if (!cur_inconsistent.empty()) { - inconsistent[*k] = cur_inconsistent; - } - if (!cur_inconsistent.empty() || !cur_missing.empty()) { - authoritative[*k] = auth->first; - } - } -} void PG::scrub_compare_maps() { @@ -3982,12 +3777,15 @@ void PG::scrub_compare_maps() maps[i] = &scrubber.received_maps[acting[i]]; } - _compare_scrubmaps( + get_pgbackend()->be_compare_scrubmaps( maps, scrubber.missing, scrubber.inconsistent, authoritative, scrubber.inconsistent_snapcolls, + scrubber.shallow_errors, + scrubber.deep_errors, + info.pgid, acting, ss); dout(2) << ss.str() << dendl; diff --git a/src/osd/PG.h b/src/osd/PG.h index af2a3ce19bb8a..23c84fddf1629 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -939,24 +939,6 @@ public: int active_pushes; void repair_object(const hobject_t& soid, ScrubMap::object *po, int bad_peer, int ok_peer); - map::const_iterator _select_auth_object( - const hobject_t &obj, - const map &maps); - - enum error_type { - CLEAN, - DEEP_ERROR, - SHALLOW_ERROR - }; - enum error_type _compare_scrub_objects(ScrubMap::object &auth, - ScrubMap::object &candidate, - ostream &errorstream); - void _compare_scrubmaps(const map &maps, - map > &missing, - map > &inconsistent, - map &authoritative, - map > &inconsistent_snapcolls, - ostream &errorstream); void scrub(ThreadPool::TPHandle &handle); void classic_scrub(ThreadPool::TPHandle &handle); void chunky_scrub(ThreadPool::TPHandle &handle); diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h index d380f5a044893..52b14908a62b0 100644 --- a/src/osd/PGBackend.h +++ b/src/osd/PGBackend.h @@ -433,6 +433,22 @@ virtual bool scrub_supported() { return false; } virtual void be_scan_list(ScrubMap &map, vector &ls, bool deep, ThreadPool::TPHandle &handle) { assert(0); } + virtual enum scrub_error_type be_compare_scrub_objects( + ScrubMap::object &auth, + ScrubMap::object &candidate, + ostream &errorstream) { assert(0); } + virtual map::const_iterator be_select_auth_object( + const hobject_t &obj, + const map &maps) { assert(0); } + virtual void be_compare_scrubmaps(const map &maps, + map > &missing, + map > &inconsistent, + map &authoritative, + map > &invalid_snapcolls, + int &shallow_errors, int &deep_errors, + const pg_t pgid, + const vector &acting, + ostream &errorstream) { assert(0); } }; #endif diff --git a/src/osd/ReplicatedBackend.cc b/src/osd/ReplicatedBackend.cc index 92b019abf9c3c..81f74c12d4bfa 100644 --- a/src/osd/ReplicatedBackend.cc +++ b/src/osd/ReplicatedBackend.cc @@ -867,3 +867,213 @@ void ReplicatedBackend::be_scan_list( } } } + +enum scrub_error_type ReplicatedBackend::be_compare_scrub_objects( + ScrubMap::object &auth, + ScrubMap::object &candidate, + ostream &errorstream) +{ + enum scrub_error_type error = CLEAN; + if (candidate.read_error) { + // This can occur on stat() of a shallow scrub, but in that case size will + // be invalid, and this will be over-ridden below. + error = DEEP_ERROR; + errorstream << "candidate had a read error"; + } + if (auth.digest_present && candidate.digest_present) { + if (auth.digest != candidate.digest) { + if (error != CLEAN) + errorstream << ", "; + error = DEEP_ERROR; + + errorstream << "digest " << candidate.digest + << " != known digest " << auth.digest; + } + } + if (auth.omap_digest_present && candidate.omap_digest_present) { + if (auth.omap_digest != candidate.omap_digest) { + if (error != CLEAN) + errorstream << ", "; + error = DEEP_ERROR; + + errorstream << "omap_digest " << candidate.omap_digest + << " != known omap_digest " << auth.omap_digest; + } + } + // Shallow error takes precendence because this will be seen by + // both types of scrubs. + if (auth.size != candidate.size) { + if (error != CLEAN) + errorstream << ", "; + error = SHALLOW_ERROR; + errorstream << "size " << candidate.size + << " != known size " << auth.size; + } + for (map::const_iterator i = auth.attrs.begin(); + i != auth.attrs.end(); + ++i) { + if (!candidate.attrs.count(i->first)) { + if (error != CLEAN) + errorstream << ", "; + error = SHALLOW_ERROR; + errorstream << "missing attr " << i->first; + } else if (candidate.attrs.find(i->first)->second.cmp(i->second)) { + if (error != CLEAN) + errorstream << ", "; + error = SHALLOW_ERROR; + errorstream << "attr value mismatch " << i->first; + } + } + for (map::const_iterator i = candidate.attrs.begin(); + i != candidate.attrs.end(); + ++i) { + if (!auth.attrs.count(i->first)) { + if (error != CLEAN) + errorstream << ", "; + error = SHALLOW_ERROR; + errorstream << "extra attr " << i->first; + } + } + return error; +} + +map::const_iterator ReplicatedBackend::be_select_auth_object( + const hobject_t &obj, + const map &maps) +{ + map::const_iterator auth = maps.end(); + for (map::const_iterator j = maps.begin(); + j != maps.end(); + ++j) { + map::iterator i = + j->second->objects.find(obj); + if (i == j->second->objects.end()) { + continue; + } + if (auth == maps.end()) { + // Something is better than nothing + // TODO: something is NOT better than nothing, do something like + // unfound_lost if no valid copies can be found, or just mark unfound + auth = j; + dout(10) << __func__ << ": selecting osd " << j->first + << " for obj " << obj + << ", auth == maps.end()" + << dendl; + continue; + } + if (i->second.read_error) { + // scrub encountered read error, probably corrupt + dout(10) << __func__ << ": rejecting osd " << j->first + << " for obj " << obj + << ", read_error" + << dendl; + continue; + } + map::iterator k = i->second.attrs.find(OI_ATTR); + if (k == i->second.attrs.end()) { + // no object info on object, probably corrupt + dout(10) << __func__ << ": rejecting osd " << j->first + << " for obj " << obj + << ", no oi attr" + << dendl; + continue; + } + bufferlist bl; + bl.push_back(k->second); + object_info_t oi; + try { + bufferlist::iterator bliter = bl.begin(); + ::decode(oi, bliter); + } catch (...) { + dout(10) << __func__ << ": rejecting osd " << j->first + << " for obj " << obj + << ", corrupt oi attr" + << dendl; + // invalid object info, probably corrupt + continue; + } + if (oi.size != i->second.size) { + // invalid size, probably corrupt + dout(10) << __func__ << ": rejecting osd " << j->first + << " for obj " << obj + << ", size mismatch" + << dendl; + // invalid object info, probably corrupt + continue; + } + dout(10) << __func__ << ": selecting osd " << j->first + << " for obj " << obj + << dendl; + auth = j; + } + return auth; +} + +void ReplicatedBackend::be_compare_scrubmaps(const map &maps, + map > &missing, + map > &inconsistent, + map &authoritative, + map > &invalid_snapcolls, + int &shallow_errors, + int &deep_errors, + const pg_t pgid, + const vector &acting, + ostream &errorstream) +{ + map::const_iterator i; + map::const_iterator j; + set master_set; + + // Construct master set + for (j = maps.begin(); j != maps.end(); ++j) { + for (i = j->second->objects.begin(); i != j->second->objects.end(); ++i) { + master_set.insert(i->first); + } + } + + // Check maps against master set and each other + for (set::const_iterator k = master_set.begin(); + k != master_set.end(); + ++k) { + map::const_iterator auth = be_select_auth_object(*k, maps); + assert(auth != maps.end()); + set cur_missing; + set cur_inconsistent; + for (j = maps.begin(); j != maps.end(); ++j) { + if (j == auth) + continue; + if (j->second->objects.count(*k)) { + // Compare + stringstream ss; + enum scrub_error_type error = be_compare_scrub_objects(auth->second->objects[*k], + j->second->objects[*k], + ss); + if (error != CLEAN) { + cur_inconsistent.insert(j->first); + if (error == SHALLOW_ERROR) + ++shallow_errors; + else + ++deep_errors; + errorstream << pgid << " osd." << acting[j->first] + << ": soid " << *k << " " << ss.str() << std::endl; + } + } else { + cur_missing.insert(j->first); + ++shallow_errors; + errorstream << pgid + << " osd." << acting[j->first] + << " missing " << *k << std::endl; + } + } + assert(auth != maps.end()); + if (!cur_missing.empty()) { + missing[*k] = cur_missing; + } + if (!cur_inconsistent.empty()) { + inconsistent[*k] = cur_inconsistent; + } + if (!cur_inconsistent.empty() || !cur_missing.empty()) { + authoritative[*k] = auth->first; + } + } +} diff --git a/src/osd/ReplicatedBackend.h b/src/osd/ReplicatedBackend.h index c53b5de911127..324607234b205 100644 --- a/src/osd/ReplicatedBackend.h +++ b/src/osd/ReplicatedBackend.h @@ -492,6 +492,22 @@ private: bool scrub_supported() { return true; } void be_scan_list(ScrubMap &map, vector &ls, bool deep, ThreadPool::TPHandle &handle); + enum scrub_error_type be_compare_scrub_objects( + ScrubMap::object &auth, + ScrubMap::object &candidate, + ostream &errorstream); + map::const_iterator be_select_auth_object( + const hobject_t &obj, + const map &maps); + void be_compare_scrubmaps(const map &maps, + map > &missing, + map > &inconsistent, + map &authoritative, + map > &invalid_snapcolls, + int &shallow_errors, int &deep_errors, + const pg_t pgid, + const vector &acting, + ostream &errorstream); }; #endif diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index e8252046041cf..e2a4822a2c497 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -3082,4 +3082,9 @@ struct obj_list_snap_response_t { WRITE_CLASS_ENCODER(obj_list_snap_response_t) +enum scrub_error_type { + CLEAN, + DEEP_ERROR, + SHALLOW_ERROR +}; #endif -- 2.39.5