From: Sage Weil <sage@redhat.com>
Date: Tue, 23 Jul 2019 18:16:53 +0000 (-0500)
Subject: osd/PeeringState: set WAIT state and block ops to wait for prior readable_until
X-Git-Tag: v15.1.0~1379^2~13
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1d0c6f53191348c91888d9585e40389d897496eb;p=ceph.git

osd/PeeringState: set WAIT state and block ops to wait for prior readable_until

If we start a new interval and the prior interval may have OSDs that
are still readable, set the WAIT state bit and block operations until
sufficient time has elapsed.

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 7a32b6ff85dd..1bae4787247c 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1411,6 +1411,7 @@ protected:
   bool is_premerge() const { return recovery_state.is_premerge(); }
   bool is_repair() const { return recovery_state.is_repair(); }
   bool is_laggy() const { return state_test(PG_STATE_LAGGY); }
+  bool is_wait() const { return state_test(PG_STATE_WAIT); }
 
   bool is_empty() const { return recovery_state.is_empty(); }
 
diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 0c354d73add1..98e70ac1f36c 100644
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -5734,6 +5734,19 @@ boost::statechart::result PeeringState::Active::react(const AllReplicasActivated
     ps->state_set(PG_STATE_ACTIVE);
   }
 
+  auto mnow = pl->get_mnow();
+  if (ps->prior_readable_until_ub > mnow) {
+    psdout(10) << " waiting for prior_readable_until_ub "
+	       << ps->prior_readable_until_ub << " > mnow " << mnow << dendl;
+    ps->state_set(PG_STATE_WAIT);
+    pl->queue_check_readable(
+      ps->last_peering_reset,
+      ps->prior_readable_until_ub - mnow);
+  } else {
+    psdout(10) << " mnow " << mnow << " >= prior_readable_until_ub "
+	       << ps->prior_readable_until_ub << dendl;
+  }
+
   if (ps->pool.info.has_flag(pg_pool_t::FLAG_CREATING)) {
     pl->send_pg_created(pgid);
   }
diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h
index 2a3910294c8f..6f9c4b1e2d63 100644
--- a/src/osd/PeeringState.h
+++ b/src/osd/PeeringState.h
@@ -1959,6 +1959,11 @@ public:
     return prior_readable_until_ub;
   }
 
+  /// Reset prior intervals' readable_until upper bound (e.g., bc it passed)
+  void clear_prior_readable_until_ub() {
+    prior_readable_until_ub = ceph::signedspan::zero();
+  }
+
   void renew_lease(ceph::signedspan now) {
     bool was_min = (readable_until_ub == readable_until);
     readable_until_ub_sent = now + readable_interval;
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 21b0f4cfff39..de99a66acb95 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -754,7 +754,9 @@ void PrimaryLogPG::maybe_force_recovery()
 
 bool PrimaryLogPG::check_laggy(OpRequestRef& op)
 {
-  if (!state_test(PG_STATE_LAGGY)) {
+  if (state_test(PG_STATE_WAIT)) {
+    dout(10) << __func__ << " PG is WAIT state" << dendl;
+  } else if (!state_test(PG_STATE_LAGGY)) {
     auto mnow = osd->get_mnow();
     auto ru = recovery_state.get_readable_until();
     if (mnow <= ru) {
@@ -777,7 +779,7 @@ bool PrimaryLogPG::check_laggy(OpRequestRef& op)
 
 bool PrimaryLogPG::check_laggy_requeue(OpRequestRef& op)
 {
-  if (!state_test(PG_STATE_LAGGY)) {
+  if (!state_test(PG_STATE_WAIT) && !state_test(PG_STATE_LAGGY)) {
     return true; // not laggy
   }
   dout(10) << __func__ << " not readable" << dendl;
@@ -788,17 +790,48 @@ bool PrimaryLogPG::check_laggy_requeue(OpRequestRef& op)
 
 void PrimaryLogPG::recheck_readable()
 {
+  if (!is_wait() && !is_laggy()) {
+    dout(20) << __func__ << " wasn't wait or laggy" << dendl;
+    return;
+  }
+  auto mnow = osd->get_mnow();
+  bool pub = false;
+  if (is_wait()) {
+    auto prior_readable_until_ub = recovery_state.get_prior_readable_until_ub();
+    if (mnow < prior_readable_until_ub) {
+      dout(10) << __func__ << " still wait (mnow " << mnow
+	       << " < prior_readable_until_ub " << prior_readable_until_ub
+	       << dendl;
+    } else {
+      dout(10) << __func__ << " no longer wait (mnow " << mnow
+	       << " >= prior_readable_until_ub " << prior_readable_until_ub
+	       << ")" << dendl;
+      state_clear(PG_STATE_WAIT);
+      recovery_state.clear_prior_readable_until_ub();
+      pub = true;
+    }
+  }
   if (is_laggy()) {
     auto ru = recovery_state.get_readable_until();
-    auto mnow = osd->get_mnow();
-    if (ru > mnow) {
-      dout(10) << __func__ << " no longer laggy (ru " << ru << " > mnow " << mnow
-	       << ")" << dendl;
+    if (ru == ceph::signedspan::zero()) {
+      dout(10) << __func__ << " still laggy (mnow " << mnow
+	       << ", readable_until zero)" << dendl;
+    } else if (mnow >= ru) {
+      dout(10) << __func__ << " still laggy (mnow " << mnow
+	       << " >= readable_until " << ru << ")" << dendl;
+    } else {
+      dout(10) << __func__ << " no longer laggy (mnow " << mnow
+	       << " < readable_until " << ru << ")" << dendl;
       state_clear(PG_STATE_LAGGY);
-      publish_stats_to_osd();
-      requeue_ops(waiting_for_readable);
+      pub = true;
     }
   }
+  if (pub) {
+    publish_stats_to_osd();
+  }
+  if (!is_laggy() && !is_wait()) {
+    requeue_ops(waiting_for_readable);
+  }
 }
 
 bool PrimaryLogPG::pgls_filter(const PGLSFilter& filter, const hobject_t& sobj)