PG: break waiting_for_peered out of waiting_for_active

waiting_for_peered now holds ops until peering completes (activation,
not necessarily state active).  waiting_for_active now holds
specifically MOSDOp blocked on:
- scrub
- replay
- state active

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 4c05a8400fae7fcdbc072891caa5e4cd6e0d6831..d6db60001f8bd8386ddb5f444ee911ed7f293936 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1795,6 +1795,7 @@ void PG::replay_queued_ops()
   replay_queue.clear();
   requeue_ops(replay);
   requeue_ops(waiting_for_active);
+  assert(waiting_for_peered.empty());
 
   publish_stats_to_osd();
 }
@@ -1827,7 +1828,7 @@ void PG::_activate_committed(epoch_t e)
     state_set(PG_STATE_ACTIVE);
     // waiters
     if (flushes_in_progress == 0) {
-      requeue_ops(waiting_for_active);
+      requeue_ops(waiting_for_peered);
     }
   }
 
@@ -2049,11 +2050,12 @@ void PG::split_ops(PG *child, unsigned split_bits) {
   assert(waiting_for_degraded_object.empty());
   assert(waiting_for_ack.empty());
   assert(waiting_for_ondisk.empty());
+  assert(waiting_for_active.empty());
   split_replay_queue(&replay_queue, &(child->replay_queue), match, split_bits);
 
-  osd->dequeue_pg(this, &waiting_for_active);
+  osd->dequeue_pg(this, &waiting_for_peered);
   OSD::split_list(
-    &waiting_for_active, &(child->waiting_for_active), match, split_bits);
+    &waiting_for_peered, &(child->waiting_for_peered), match, split_bits);
   {
     Mutex::Locker l(map_lock); // to avoid a race with the osd dispatch
     OSD::split_list(
@@ -4768,7 +4770,7 @@ void PG::start_peering_interval(
     on_role_change();
 
     // take active waiters
-    requeue_ops(waiting_for_active);
+    requeue_ops(waiting_for_peered);
 
   } else {
     // no role change.
@@ -6507,7 +6509,7 @@ boost::statechart::result PG::RecoveryState::Active::react(const AllReplicasActi
 
   // waiters
   if (!pg->is_replay() && pg->flushes_in_progress == 0) {
-    pg->requeue_ops(pg->waiting_for_active);
+    pg->requeue_ops(pg->waiting_for_peered);
   }
 
   pg->on_activate();

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 50f9b24f75c5a5374e2f31302c682fdba0bd2243..4408b8ca8ba71fd2da9e6a024f3c6c96511cc787 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -729,7 +729,12 @@ protected:
   // pg waiters
   unsigned flushes_in_progress;
 
+  // ops waiting on peered
+  list<OpRequestRef>            waiting_for_peered;
+
+  // ops waiting on active (require peered as well)
   list<OpRequestRef>            waiting_for_active;
+
   list<OpRequestRef>            waiting_for_cache_not_full;
   list<OpRequestRef>            waiting_for_all_missing;
   map<hobject_t, list<OpRequestRef> > waiting_for_unreadable_object,

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index c0fca587d74bc766c3e9a613ad762bd33aa9ed82..67dd16423dbe7ac1cdb3b3f3d34e836f3c9d421a 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -1268,20 +1268,18 @@ void ReplicatedPG::do_request(
     dout(20) << flushes_in_progress
             << " flushes_in_progress pending "
             << "waiting for active on " << op << dendl;
-    waiting_for_active.push_back(op);
-    op->mark_delayed("waiting for flushes");
+    waiting_for_peered.push_back(op);
     return;
   }
 
-  if (!is_active()) {
+  if (!is_peered()) {
     // Delay unless PGBackend says it's ok
     if (pgbackend->can_handle_while_inactive(op)) {
       bool handled = pgbackend->handle_message(op);
       assert(handled);
       return;
     } else {
-      waiting_for_active.push_back(op);
-      op->mark_delayed("waiting for active");
+      waiting_for_peered.push_back(op);
       return;
     }
   }
@@ -1292,6 +1290,11 @@ void ReplicatedPG::do_request(
 
   switch (op->get_req()->get_type()) {
   case CEPH_MSG_OSD_OP:
+    if (!is_active()) {
+      dout(20) << " peered, not active, waiting for active on " << op << dendl;
+      waiting_for_active.push_back(op);
+      return;
+    }
     if (is_replay()) {
       dout(20) << " replay, waiting for active on " << op << dendl;
       waiting_for_active.push_back(op);
@@ -2398,8 +2401,8 @@ void ReplicatedPG::do_sub_op(OpRequestRef op)
     first = &m->ops[0];
   }
 
-  if (!is_active()) {
-    waiting_for_active.push_back(op);
+  if (!is_peered()) {
+    waiting_for_peered.push_back(op);
     op->mark_delayed("waiting for active");
     return;
   }
@@ -10171,7 +10174,7 @@ void ReplicatedPG::on_flushed()
   assert(flushes_in_progress > 0);
   flushes_in_progress--;
   if (flushes_in_progress == 0) {
-    requeue_ops(waiting_for_active);
+    requeue_ops(waiting_for_peered);
   }
   if (!is_active() || !is_primary()) {
     pair<hobject_t, ObjectContextRef> i;
@@ -10295,7 +10298,11 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t)
 
   // requeue everything in the reverse order they should be
   // reexamined.
+  requeue_ops(waiting_for_peered);
+
   clear_scrub_reserved();
+
+  // requeues waiting_for_active
   scrub_clear_state();
 
   context_registry_on_change();