osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit bb2bcb95f51abc206e005e44ef383ee45b8f2209)

Conflicts:
	src/osd/PG.cc (trivial, no MBackfillReserve::CANCEL)
	src/osd/PG.h (trivial)

fixup! osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions

diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh
index 2f8769abb074d49e4e6a898f6385b516c679e719..dffa7965f5733c87d581ea1ab567d04ae8d927e8 100755 (executable)
--- a/qa/standalone/erasure-code/test-erasure-eio.sh
+++ b/qa/standalone/erasure-code/test-erasure-eio.sh
@@ -415,8 +415,8 @@ function TEST_ec_recovery_errors() {
     delete_pool $poolname
 }
 
-# Test backfill with errors present
-function TEST_ec_backfill_errors() {
+# Test backfill with unfound object
+function TEST_ec_backfill_unfound() {
     local dir=$1
     local objname=myobject
     local lastobj=300
@@ -456,13 +456,14 @@ function TEST_ec_backfill_errors() {
 
     sleep 15
 
-    while(true); do
+    for tmp in $(seq 1 100); do
       state=$(get_state 2.0)
-      echo $state | grep -v backfilling
+      echo $state | grep backfill_unfound
       if [ "$?" = "0" ]; then
         break
       fi
-      echo -n "$state "
+      echo $state
+      sleep 1
     done
 
     ceph pg dump pgs
@@ -492,8 +493,8 @@ function TEST_ec_backfill_errors() {
     delete_pool $poolname
 }
 
-# Test recovery with errors present
-function TEST_ec_recovery_errors() {
+# Test recovery with unfound object
+function TEST_ec_recovery_unfound() {
     local dir=$1
     local objname=myobject
     local lastobj=100
@@ -531,13 +532,14 @@ function TEST_ec_recovery_errors() {
 
     sleep 15
 
-    while(true); do
+    for tmp in $(seq 1 100); do
       state=$(get_state 2.0)
-      echo $state | grep -v recovering
+      echo $state | grep recovery_unfound
       if [ "$?" = "0" ]; then
         break
       fi
-      echo -n "$state "
+      echo "$state "
+      sleep 1
     done
 
     ceph pg dump pgs

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 139a9ac6d17941d835bfecbea14b2e2577d3f072..3a919cdbab8a1b45a4f3ff032ad9a133ca15ba7e 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -6433,6 +6433,36 @@ PG::RecoveryState::Backfilling::react(const DeferBackfill &c)
   return transit<NotBackfilling>();
 }
 
+boost::statechart::result
+PG::RecoveryState::Backfilling::react(const UnfoundBackfill &c)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  ldout(pg->cct, 10) << "backfill has unfound, can't continue" << dendl;
+  pg->osd->local_reserver.cancel_reservation(pg->info.pgid);
+
+  pg->state_clear(PG_STATE_BACKFILLING);
+
+  for (set<pg_shard_t>::iterator it = pg->backfill_targets.begin();
+       it != pg->backfill_targets.end();
+       ++it) {
+    assert(*it != pg->pg_whoami);
+    ConnectionRef con = pg->osd->get_con_osd_cluster(
+      it->osd, pg->get_osdmap()->get_epoch());
+    if (con) {
+      pg->osd->send_message_osd_cluster(
+        new MBackfillReserve(
+         MBackfillReserve::REJECT,
+         spg_t(pg->info.pgid.pgid, it->shard),
+         pg->get_osdmap()->get_epoch()),
+       con.get());
+    }
+  }
+
+  pg->waiting_on_backfill.clear();
+
+  return transit<NotBackfilling>();
+}
+
 boost::statechart::result
 PG::RecoveryState::Backfilling::react(const RemoteReservationRejected &)
 {
@@ -6614,6 +6644,7 @@ void PG::RecoveryState::NotBackfilling::exit()
 {
   context< RecoveryMachine >().log_exit(state_name, enter_time);
   PG *pg = context< RecoveryMachine >().pg;
+  pg->state_clear(PG_STATE_UNFOUND);
   utime_t dur = ceph_clock_now() - enter_time;
   pg->osd->recoverystate_perf->tinc(rs_notbackfilling_latency, dur);
 }
@@ -6632,6 +6663,7 @@ void PG::RecoveryState::NotRecovering::exit()
 {
   context< RecoveryMachine >().log_exit(state_name, enter_time);
   PG *pg = context< RecoveryMachine >().pg;
+  pg->state_clear(PG_STATE_UNFOUND);
   utime_t dur = ceph_clock_now() - enter_time;
   pg->osd->recoverystate_perf->tinc(rs_notrecovering_latency, dur);
 }
@@ -7008,6 +7040,17 @@ PG::RecoveryState::Recovering::react(const DeferRecovery &evt)
   return transit<NotRecovering>();
 }
 
+boost::statechart::result
+PG::RecoveryState::Recovering::react(const UnfoundRecovery &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  ldout(pg->cct, 10) << "recovery has unfound, can't continue" << dendl;
+  pg->state_clear(PG_STATE_RECOVERING);
+  pg->osd->local_reserver.cancel_reservation(pg->info.pgid);
+  release_reservations(true);
+  return transit<NotRecovering>();
+}
+
 void PG::RecoveryState::Recovering::exit()
 {
   context< RecoveryMachine >().log_exit(state_name, enter_time);

diff --git a/src/osd/PG.h b/src/osd/PG.h
index fd5f36aec0662f9590ce99138268e3718dd6079b..8adb0638994047be581f45f26f9513e11f093a1f 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1577,7 +1577,19 @@ public:
       *out << "DeferRecovery: delay " << delay;
     }
   };
-
+  struct UnfoundBackfill : boost::statechart::event<UnfoundBackfill> {
+    explicit UnfoundBackfill() {}
+    void print(std::ostream *out) const {
+      *out << "UnfoundBackfill";
+    }
+  };
+  struct UnfoundRecovery : boost::statechart::event<UnfoundRecovery> {
+    explicit UnfoundRecovery() {}
+    void print(std::ostream *out) const {
+      *out << "UnfoundRecovery";
+    }
+  };
+protected:
   TrivialEvent(Initialize)
   TrivialEvent(Load)
   TrivialEvent(GotInfo)
@@ -1855,7 +1867,9 @@ public:
        boost::statechart::custom_reaction< Backfilled >,
        boost::statechart::custom_reaction< AllReplicasActivated >,
        boost::statechart::custom_reaction< DeferRecovery >,
-       boost::statechart::custom_reaction< DeferBackfill >
+       boost::statechart::custom_reaction< DeferBackfill >,
+       boost::statechart::custom_reaction< UnfoundRecovery >,
+       boost::statechart::custom_reaction< UnfoundBackfill >
        > reactions;
       boost::statechart::result react(const QueryState& q);
       boost::statechart::result react(const ActMap&);
@@ -1873,6 +1887,12 @@ public:
       boost::statechart::result react(const DeferBackfill& evt) {
        return discard_event();
       }
+      boost::statechart::result react(const UnfoundRecovery& evt) {
+       return discard_event();
+      }
+      boost::statechart::result react(const UnfoundBackfill& evt) {
+       return discard_event();
+      }
     };
 
     struct Clean : boost::statechart::state< Clean, Active >, NamedState {
@@ -1901,11 +1921,13 @@ public:
       typedef boost::mpl::list<
        boost::statechart::transition< Backfilled, Recovered >,
        boost::statechart::custom_reaction< DeferBackfill >,
+       boost::statechart::custom_reaction< UnfoundBackfill >,
        boost::statechart::custom_reaction< RemoteReservationRejected >
        > reactions;
       explicit Backfilling(my_context ctx);
       boost::statechart::result react(const RemoteReservationRejected& evt);
       boost::statechart::result react(const DeferBackfill& evt);
+      boost::statechart::result react(const UnfoundBackfill& evt);
       void exit();
     };
 
@@ -1945,13 +1967,18 @@ public:
     struct NotRecovering : boost::statechart::state< NotRecovering, Active>, NamedState {
       typedef boost::mpl::list<
        boost::statechart::transition< DoRecovery, WaitLocalRecoveryReserved >,
-       boost::statechart::custom_reaction< DeferRecovery >
+       boost::statechart::custom_reaction< DeferRecovery >,
+       boost::statechart::custom_reaction< UnfoundRecovery >
        > reactions;
       explicit NotRecovering(my_context ctx);
       boost::statechart::result react(const DeferRecovery& evt) {
        /* no-op */
        return discard_event();
       }
+      boost::statechart::result react(const UnfoundRecovery& evt) {
+       /* no-op */
+       return discard_event();
+      }
       void exit();
     };
 
@@ -1968,7 +1995,9 @@ public:
        boost::statechart::custom_reaction< MLogRec >,
        boost::statechart::custom_reaction< Activate >,
        boost::statechart::custom_reaction< DeferRecovery >,
-       boost::statechart::custom_reaction< DeferBackfill >
+       boost::statechart::custom_reaction< DeferBackfill >,
+       boost::statechart::custom_reaction< UnfoundRecovery >,
+       boost::statechart::custom_reaction< UnfoundBackfill >
        > reactions;
       boost::statechart::result react(const QueryState& q);
       boost::statechart::result react(const MInfoRec& infoevt);
@@ -1982,6 +2011,12 @@ public:
       boost::statechart::result react(const DeferBackfill& evt) {
        return discard_event();
       }
+      boost::statechart::result react(const UnfoundRecovery& evt) {
+       return discard_event();
+      }
+      boost::statechart::result react(const UnfoundBackfill& evt) {
+       return discard_event();
+      }
     };
 
     struct RepRecovering : boost::statechart::state< RepRecovering, ReplicaActive >, NamedState {
@@ -2049,6 +2084,7 @@ public:
       typedef boost::mpl::list <
        boost::statechart::custom_reaction< AllReplicasRecovered >,
        boost::statechart::custom_reaction< DeferRecovery >,
+       boost::statechart::custom_reaction< UnfoundRecovery >,
        boost::statechart::custom_reaction< RequestBackfill >
        > reactions;
       explicit Recovering(my_context ctx);
@@ -2056,6 +2092,7 @@ public:
       void release_reservations(bool cancel = false);
       boost::statechart::result react(const AllReplicasRecovered &evt);
       boost::statechart::result react(const DeferRecovery& evt);
+      boost::statechart::result react(const UnfoundRecovery& evt);
       boost::statechart::result react(const RequestBackfill &evt);
     };