From: David Zafman Date: Mon, 9 Oct 2017 15:17:29 +0000 (-0700) Subject: osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions X-Git-Tag: v12.2.3~146^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e40e2aa53b192a86abaa95fc74f05b5575e16d62;p=ceph.git osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions Signed-off-by: David Zafman (cherry picked from commit bb2bcb95f51abc206e005e44ef383ee45b8f2209) Conflicts: src/osd/PG.cc (trivial, no MBackfillReserve::CANCEL) src/osd/PG.h (trivial) fixup! osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions --- diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh index 2f8769abb07..dffa7965f57 100755 --- a/qa/standalone/erasure-code/test-erasure-eio.sh +++ b/qa/standalone/erasure-code/test-erasure-eio.sh @@ -415,8 +415,8 @@ function TEST_ec_recovery_errors() { delete_pool $poolname } -# Test backfill with errors present -function TEST_ec_backfill_errors() { +# Test backfill with unfound object +function TEST_ec_backfill_unfound() { local dir=$1 local objname=myobject local lastobj=300 @@ -456,13 +456,14 @@ function TEST_ec_backfill_errors() { sleep 15 - while(true); do + for tmp in $(seq 1 100); do state=$(get_state 2.0) - echo $state | grep -v backfilling + echo $state | grep backfill_unfound if [ "$?" = "0" ]; then break fi - echo -n "$state " + echo $state + sleep 1 done ceph pg dump pgs @@ -492,8 +493,8 @@ function TEST_ec_backfill_errors() { delete_pool $poolname } -# Test recovery with errors present -function TEST_ec_recovery_errors() { +# Test recovery with unfound object +function TEST_ec_recovery_unfound() { local dir=$1 local objname=myobject local lastobj=100 @@ -531,13 +532,14 @@ function TEST_ec_recovery_errors() { sleep 15 - while(true); do + for tmp in $(seq 1 100); do state=$(get_state 2.0) - echo $state | grep -v recovering + echo $state | grep recovery_unfound if [ "$?" = "0" ]; then break fi - echo -n "$state " + echo "$state " + sleep 1 done ceph pg dump pgs diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 139a9ac6d17..3a919cdbab8 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -6433,6 +6433,36 @@ PG::RecoveryState::Backfilling::react(const DeferBackfill &c) return transit(); } +boost::statechart::result +PG::RecoveryState::Backfilling::react(const UnfoundBackfill &c) +{ + PG *pg = context< RecoveryMachine >().pg; + ldout(pg->cct, 10) << "backfill has unfound, can't continue" << dendl; + pg->osd->local_reserver.cancel_reservation(pg->info.pgid); + + pg->state_clear(PG_STATE_BACKFILLING); + + for (set::iterator it = pg->backfill_targets.begin(); + it != pg->backfill_targets.end(); + ++it) { + assert(*it != pg->pg_whoami); + ConnectionRef con = pg->osd->get_con_osd_cluster( + it->osd, pg->get_osdmap()->get_epoch()); + if (con) { + pg->osd->send_message_osd_cluster( + new MBackfillReserve( + MBackfillReserve::REJECT, + spg_t(pg->info.pgid.pgid, it->shard), + pg->get_osdmap()->get_epoch()), + con.get()); + } + } + + pg->waiting_on_backfill.clear(); + + return transit(); +} + boost::statechart::result PG::RecoveryState::Backfilling::react(const RemoteReservationRejected &) { @@ -6614,6 +6644,7 @@ void PG::RecoveryState::NotBackfilling::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; + pg->state_clear(PG_STATE_UNFOUND); utime_t dur = ceph_clock_now() - enter_time; pg->osd->recoverystate_perf->tinc(rs_notbackfilling_latency, dur); } @@ -6632,6 +6663,7 @@ void PG::RecoveryState::NotRecovering::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; + pg->state_clear(PG_STATE_UNFOUND); utime_t dur = ceph_clock_now() - enter_time; pg->osd->recoverystate_perf->tinc(rs_notrecovering_latency, dur); } @@ -7008,6 +7040,17 @@ PG::RecoveryState::Recovering::react(const DeferRecovery &evt) return transit(); } +boost::statechart::result +PG::RecoveryState::Recovering::react(const UnfoundRecovery &evt) +{ + PG *pg = context< RecoveryMachine >().pg; + ldout(pg->cct, 10) << "recovery has unfound, can't continue" << dendl; + pg->state_clear(PG_STATE_RECOVERING); + pg->osd->local_reserver.cancel_reservation(pg->info.pgid); + release_reservations(true); + return transit(); +} + void PG::RecoveryState::Recovering::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); diff --git a/src/osd/PG.h b/src/osd/PG.h index fd5f36aec06..8adb0638994 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -1577,7 +1577,19 @@ public: *out << "DeferRecovery: delay " << delay; } }; - + struct UnfoundBackfill : boost::statechart::event { + explicit UnfoundBackfill() {} + void print(std::ostream *out) const { + *out << "UnfoundBackfill"; + } + }; + struct UnfoundRecovery : boost::statechart::event { + explicit UnfoundRecovery() {} + void print(std::ostream *out) const { + *out << "UnfoundRecovery"; + } + }; +protected: TrivialEvent(Initialize) TrivialEvent(Load) TrivialEvent(GotInfo) @@ -1855,7 +1867,9 @@ public: boost::statechart::custom_reaction< Backfilled >, boost::statechart::custom_reaction< AllReplicasActivated >, boost::statechart::custom_reaction< DeferRecovery >, - boost::statechart::custom_reaction< DeferBackfill > + boost::statechart::custom_reaction< DeferBackfill >, + boost::statechart::custom_reaction< UnfoundRecovery >, + boost::statechart::custom_reaction< UnfoundBackfill > > reactions; boost::statechart::result react(const QueryState& q); boost::statechart::result react(const ActMap&); @@ -1873,6 +1887,12 @@ public: boost::statechart::result react(const DeferBackfill& evt) { return discard_event(); } + boost::statechart::result react(const UnfoundRecovery& evt) { + return discard_event(); + } + boost::statechart::result react(const UnfoundBackfill& evt) { + return discard_event(); + } }; struct Clean : boost::statechart::state< Clean, Active >, NamedState { @@ -1901,11 +1921,13 @@ public: typedef boost::mpl::list< boost::statechart::transition< Backfilled, Recovered >, boost::statechart::custom_reaction< DeferBackfill >, + boost::statechart::custom_reaction< UnfoundBackfill >, boost::statechart::custom_reaction< RemoteReservationRejected > > reactions; explicit Backfilling(my_context ctx); boost::statechart::result react(const RemoteReservationRejected& evt); boost::statechart::result react(const DeferBackfill& evt); + boost::statechart::result react(const UnfoundBackfill& evt); void exit(); }; @@ -1945,13 +1967,18 @@ public: struct NotRecovering : boost::statechart::state< NotRecovering, Active>, NamedState { typedef boost::mpl::list< boost::statechart::transition< DoRecovery, WaitLocalRecoveryReserved >, - boost::statechart::custom_reaction< DeferRecovery > + boost::statechart::custom_reaction< DeferRecovery >, + boost::statechart::custom_reaction< UnfoundRecovery > > reactions; explicit NotRecovering(my_context ctx); boost::statechart::result react(const DeferRecovery& evt) { /* no-op */ return discard_event(); } + boost::statechart::result react(const UnfoundRecovery& evt) { + /* no-op */ + return discard_event(); + } void exit(); }; @@ -1968,7 +1995,9 @@ public: boost::statechart::custom_reaction< MLogRec >, boost::statechart::custom_reaction< Activate >, boost::statechart::custom_reaction< DeferRecovery >, - boost::statechart::custom_reaction< DeferBackfill > + boost::statechart::custom_reaction< DeferBackfill >, + boost::statechart::custom_reaction< UnfoundRecovery >, + boost::statechart::custom_reaction< UnfoundBackfill > > reactions; boost::statechart::result react(const QueryState& q); boost::statechart::result react(const MInfoRec& infoevt); @@ -1982,6 +2011,12 @@ public: boost::statechart::result react(const DeferBackfill& evt) { return discard_event(); } + boost::statechart::result react(const UnfoundRecovery& evt) { + return discard_event(); + } + boost::statechart::result react(const UnfoundBackfill& evt) { + return discard_event(); + } }; struct RepRecovering : boost::statechart::state< RepRecovering, ReplicaActive >, NamedState { @@ -2049,6 +2084,7 @@ public: typedef boost::mpl::list < boost::statechart::custom_reaction< AllReplicasRecovered >, boost::statechart::custom_reaction< DeferRecovery >, + boost::statechart::custom_reaction< UnfoundRecovery >, boost::statechart::custom_reaction< RequestBackfill > > reactions; explicit Recovering(my_context ctx); @@ -2056,6 +2092,7 @@ public: void release_reservations(bool cancel = false); boost::statechart::result react(const AllReplicasRecovered &evt); boost::statechart::result react(const DeferRecovery& evt); + boost::statechart::result react(const UnfoundRecovery& evt); boost::statechart::result react(const RequestBackfill &evt); };