osd/PeeringState: set WAIT state and block ops to wait for prior readable_until

If we start a new interval and the prior interval may have OSDs that
are still readable, set the WAIT state bit and block operations until
sufficient time has elapsed.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 7a32b6ff85dd6c00b8473dbdee7af28a9e4ba6ae..1bae4787247c373f5685b5d5dc4139a212dd4f5a 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1411,6 +1411,7 @@ protected:
   bool is_premerge() const { return recovery_state.is_premerge(); }
   bool is_repair() const { return recovery_state.is_repair(); }
   bool is_laggy() const { return state_test(PG_STATE_LAGGY); }
+  bool is_wait() const { return state_test(PG_STATE_WAIT); }
 
   bool is_empty() const { return recovery_state.is_empty(); }
 

diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 0c354d73add1c954f31b09c6c47642e0723eab90..98e70ac1f36ca3014680d32019125983a0b33fbb 100644 (file)
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -5734,6 +5734,19 @@ boost::statechart::result PeeringState::Active::react(const AllReplicasActivated
     ps->state_set(PG_STATE_ACTIVE);
   }
 
+  auto mnow = pl->get_mnow();
+  if (ps->prior_readable_until_ub > mnow) {
+    psdout(10) << " waiting for prior_readable_until_ub "
+              << ps->prior_readable_until_ub << " > mnow " << mnow << dendl;
+    ps->state_set(PG_STATE_WAIT);
+    pl->queue_check_readable(
+      ps->last_peering_reset,
+      ps->prior_readable_until_ub - mnow);
+  } else {
+    psdout(10) << " mnow " << mnow << " >= prior_readable_until_ub "
+              << ps->prior_readable_until_ub << dendl;
+  }
+
   if (ps->pool.info.has_flag(pg_pool_t::FLAG_CREATING)) {
     pl->send_pg_created(pgid);
   }

diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h
index 2a3910294c8fe20c4011e6310c1b652c1707f6cf..6f9c4b1e2d63e5668a28794c7f8c3427affd7e0e 100644 (file)
--- a/src/osd/PeeringState.h
+++ b/src/osd/PeeringState.h
@@ -1959,6 +1959,11 @@ public:
     return prior_readable_until_ub;
   }
 
+  /// Reset prior intervals' readable_until upper bound (e.g., bc it passed)
+  void clear_prior_readable_until_ub() {
+    prior_readable_until_ub = ceph::signedspan::zero();
+  }
+
   void renew_lease(ceph::signedspan now) {
     bool was_min = (readable_until_ub == readable_until);
     readable_until_ub_sent = now + readable_interval;

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 21b0f4cfff3983093eb11eecf88c6d0addd954dd..de99a66acb95ad525f990d48090cc8c44a79dedb 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -754,7 +754,9 @@ void PrimaryLogPG::maybe_force_recovery()
 
 bool PrimaryLogPG::check_laggy(OpRequestRef& op)
 {
-  if (!state_test(PG_STATE_LAGGY)) {
+  if (state_test(PG_STATE_WAIT)) {
+    dout(10) << __func__ << " PG is WAIT state" << dendl;
+  } else if (!state_test(PG_STATE_LAGGY)) {
     auto mnow = osd->get_mnow();
     auto ru = recovery_state.get_readable_until();
     if (mnow <= ru) {
@@ -777,7 +779,7 @@ bool PrimaryLogPG::check_laggy(OpRequestRef& op)
 
 bool PrimaryLogPG::check_laggy_requeue(OpRequestRef& op)
 {
-  if (!state_test(PG_STATE_LAGGY)) {
+  if (!state_test(PG_STATE_WAIT) && !state_test(PG_STATE_LAGGY)) {
     return true; // not laggy
   }
   dout(10) << __func__ << " not readable" << dendl;
@@ -788,17 +790,48 @@ bool PrimaryLogPG::check_laggy_requeue(OpRequestRef& op)
 
 void PrimaryLogPG::recheck_readable()
 {
+  if (!is_wait() && !is_laggy()) {
+    dout(20) << __func__ << " wasn't wait or laggy" << dendl;
+    return;
+  }
+  auto mnow = osd->get_mnow();
+  bool pub = false;
+  if (is_wait()) {
+    auto prior_readable_until_ub = recovery_state.get_prior_readable_until_ub();
+    if (mnow < prior_readable_until_ub) {
+      dout(10) << __func__ << " still wait (mnow " << mnow
+              << " < prior_readable_until_ub " << prior_readable_until_ub
+              << dendl;
+    } else {
+      dout(10) << __func__ << " no longer wait (mnow " << mnow
+              << " >= prior_readable_until_ub " << prior_readable_until_ub
+              << ")" << dendl;
+      state_clear(PG_STATE_WAIT);
+      recovery_state.clear_prior_readable_until_ub();
+      pub = true;
+    }
+  }
   if (is_laggy()) {
     auto ru = recovery_state.get_readable_until();
-    auto mnow = osd->get_mnow();
-    if (ru > mnow) {
-      dout(10) << __func__ << " no longer laggy (ru " << ru << " > mnow " << mnow
-              << ")" << dendl;
+    if (ru == ceph::signedspan::zero()) {
+      dout(10) << __func__ << " still laggy (mnow " << mnow
+              << ", readable_until zero)" << dendl;
+    } else if (mnow >= ru) {
+      dout(10) << __func__ << " still laggy (mnow " << mnow
+              << " >= readable_until " << ru << ")" << dendl;
+    } else {
+      dout(10) << __func__ << " no longer laggy (mnow " << mnow
+              << " < readable_until " << ru << ")" << dendl;
       state_clear(PG_STATE_LAGGY);
-      publish_stats_to_osd();
-      requeue_ops(waiting_for_readable);
+      pub = true;
     }
   }
+  if (pub) {
+    publish_stats_to_osd();
+  }
+  if (!is_laggy() && !is_wait()) {
+    requeue_ops(waiting_for_readable);
+  }
 }
 
 bool PrimaryLogPG::pgls_filter(const PGLSFilter& filter, const hobject_t& sobj)