From 202606c26dadc5f7e65a8aed69fbc11fb0767405 Mon Sep 17 00:00:00 2001 From: xie xingguo Date: Mon, 3 Dec 2018 16:54:19 +0800 Subject: [PATCH] osd: automatically repair replicated replica on pulling error However this is not a very complete solution since the broken object info may still get lost if we switch primaries or simply power off nodes. I think a better idea would be also adding these kind of broken objects back into replica's own missing set simultaneously, e.g., like we handling primary reading errors. But for now I am not sure if that should be a concern? Fixes: http://tracker.ceph.com/issues/39101 Signed-off-by: xie xingguo --- src/osd/PGBackend.h | 4 +++- src/osd/PrimaryLogPG.cc | 13 +++++++++++-- src/osd/PrimaryLogPG.h | 4 +++- src/osd/ReplicatedBackend.cc | 6 ++++-- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h index fa1354c70b269..b48cbfb1bd4af 100644 --- a/src/osd/PGBackend.h +++ b/src/osd/PGBackend.h @@ -105,7 +105,9 @@ typedef std::shared_ptr OSDMapRef; pg_shard_t peer, const hobject_t oid) = 0; - virtual void failed_push(const list &from, const hobject_t &soid) = 0; + virtual void failed_push(const list &from, + const hobject_t &soid, + const eversion_t &need = eversion_t()) = 0; virtual void finish_degraded_object(const hobject_t& oid) = 0; virtual void primary_failed(const hobject_t &soid) = 0; virtual bool primary_error(const hobject_t& soid, eversion_t v) = 0; diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index abf7e985294e8..efda4ad049153 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -11694,7 +11694,8 @@ void PrimaryLogPG::primary_failed(const hobject_t &soid) failed_push(fl, soid); } -void PrimaryLogPG::failed_push(const list &from, const hobject_t &soid) +void PrimaryLogPG::failed_push(const list &from, + const hobject_t &soid, const eversion_t &need) { dout(20) << __func__ << ": " << soid << dendl; ceph_assert(recovering.count(soid)); @@ -11705,8 +11706,16 @@ void PrimaryLogPG::failed_push(const list &from, const hobject_t &so requeue_ops(blocked_ops); } recovering.erase(soid); - for (auto&& i : from) + for (auto&& i : from) { missing_loc.remove_location(soid, i); + if (need != eversion_t()) { + dout(0) << __func__ << " adding " << soid << " to shard " << i + << "'s missing set too" << dendl; + auto pm = peer_missing.find(i); + if (pm != peer_missing.end()) + pm->second.add(soid, need, eversion_t(), false); + } + } dout(0) << __func__ << " " << soid << " from shard " << from << ", reps on " << missing_loc.get_locations(soid) << " unfound? " << missing_loc.is_unfound(soid) << dendl; diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h index c0f4afb1846ad..9aed6243b1ec7 100644 --- a/src/osd/PrimaryLogPG.h +++ b/src/osd/PrimaryLogPG.h @@ -290,7 +290,9 @@ public: const hobject_t &oid, const object_stat_sum_t &stat_diff, bool is_delete) override; - void failed_push(const list &from, const hobject_t &soid) override; + void failed_push(const list &from, + const hobject_t &soid, + const eversion_t &need = eversion_t()) override; void primary_failed(const hobject_t &soid) override; bool primary_error(const hobject_t& soid, eversion_t v) override; void cancel_pull(const hobject_t &soid) override; diff --git a/src/osd/ReplicatedBackend.cc b/src/osd/ReplicatedBackend.cc index 9614a58997343..23bc3b9bf55db 100644 --- a/src/osd/ReplicatedBackend.cc +++ b/src/osd/ReplicatedBackend.cc @@ -2182,9 +2182,11 @@ void ReplicatedBackend::_failed_pull(pg_shard_t from, const hobject_t &soid) { dout(20) << __func__ << ": " << soid << " from " << from << dendl; list fl = { from }; - get_parent()->failed_push(fl, soid); + auto it = pulling.find(soid); + assert(it != pulling.end()); + get_parent()->failed_push(fl, soid, it->second.recovery_info.version); - clear_pull(pulling.find(soid)); + clear_pull(it); } void ReplicatedBackend::clear_pull_from( -- 2.39.5