osd: move PG peering waiters into op wq

This resolves problems with a peering event being delivered triggering
advance_pg which triggers a requeue of waiting events that are requeued
*behind* the event we are processing.  It also reduces the number of
wait lists by one, yay!

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a7bb216de8f7d31c631401be32d75ab460703eae..356d5f1734ae2e13452075a06f21952aac0b391a 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -9661,13 +9661,13 @@ void OSD::ShardedOpWQ::_add_slot_waiter(
 {
   if (qi.is_peering()) {
     dout(20) << __func__ << " " << pgid
-            << " no pg, peering, item epoch is "
+            << " peering, item epoch is "
             << qi.get_map_epoch()
             << ", will wait on " << qi << dendl;
     slot.waiting_peering[qi.get_map_epoch()].push_back(std::move(qi));
   } else {
     dout(20) << __func__ << " " << pgid
-            << " no pg, item epoch is "
+            << " item epoch is "
             << qi.get_map_epoch()
             << ", will wait on " << qi << dendl;
     slot.waiting.push_back(std::move(qi));
@@ -9893,6 +9893,15 @@ void OSD::ShardedOpWQ::_process(uint32_t thread_index, heartbeat_handle_d *hb)
     sdata->sdata_op_ordering_lock.Unlock();
     return;
   }
+  if (qi.is_peering()) {
+    OSDMapRef osdmap = sdata->waiting_for_pg_osdmap;
+    if (qi.get_map_epoch() > osdmap->get_epoch()) {
+      _add_slot_waiter(token, slot, std::move(qi));
+      sdata->sdata_op_ordering_lock.Unlock();
+      pg->unlock();
+      return;
+    }
+  }
   sdata->sdata_op_ordering_lock.Unlock();
 
   if (pushes_to_free) {

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 320f3a320252aeec1140ca98097573866e42ca30..26edecd6642971d76c570c002dfab5d372638999 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -6305,21 +6305,13 @@ void PG::take_waiters()
 {
   dout(10) << "take_waiters" << dendl;
   requeue_map_waiters();
-  for (auto i = peering_waiters.rbegin();
-       i != peering_waiters.rend();
-       ++i) {
-    osd->osd->enqueue_peering_evt_front(info.pgid, *i);
-  }
-  peering_waiters.clear();
 }
 
 void PG::do_peering_event(PGPeeringEventRef evt, RecoveryCtx *rctx)
 {
   dout(10) << __func__ << ": " << evt->get_desc() << dendl;
-  if (!have_same_or_newer_map(evt->get_epoch_sent())) {
-    dout(10) << "deferring event " << evt->get_desc() << dendl;
-    peering_waiters.push_back(evt);
-  } else if (old_peering_evt(evt)) {
+  assert(have_same_or_newer_map(evt->get_epoch_sent()));
+  if (old_peering_evt(evt)) {
     dout(10) << "discard old " << evt->get_desc() << dendl;
   } else {
     recovery_state.handle_event(evt, rctx);

diff --git a/src/osd/PG.h b/src/osd/PG.h
index ea577e07f71568ea89a8ece603710a9ed5034918..bf9aaae3f409d0e6c6bc9fb19268003784c4a8a7 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1792,8 +1792,6 @@ protected:
   };
 
 
-  list<PGPeeringEventRef> peering_waiters;
-
   struct QueryState : boost::statechart::event< QueryState > {
     Formatter *f;
     explicit QueryState(Formatter *f) : f(f) {}