PG: replica_scrub also should not block

As with scrub, replica scrub wait()ed for last_update_complete to catch
up to last_update.  Now, it will requeue the message when that condition
is satisfied.

Signed-off-by: Samuel Just <samuel.just@dreamhost.com>

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 149b722238eb4c177b2a8eafa047f1c1e4a3470f..6130c9a63c7eab36bbb215f8c0d6e1e354e27b9f 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -3078,26 +3078,43 @@ void PG::repair_object(const sobject_t& soid, ScrubMap::object *po, int bad_peer
   osd->queue_for_recovery(this);
 }
 
+/* replica_scrub
+ *
+ * If msg->scrub_from is not set, replica_scrub calls build_scrubmap to
+ * build a complete map (with the pg lock dropped).
+ *
+ * If msg->scrub_from is set, replica_scrub sets finalizing_scrub.
+ * Similarly to scrub, if last_update_applied is behind info.last_update
+ * replica_scrub returns to be requeued by sub_op_modify_applied.
+ * replica_scrub then builds an incremental scrub map with the 
+ * pg lock held.
+ */
 void PG::replica_scrub(MOSDRepScrub *msg)
 {
+  assert(!active_rep_scrub);
   dout(7) << "replica_scrub" << dendl;
 
   if (msg->map_epoch < info.history.same_acting_since) {
-    dout(10) << "replica_scrub discarding old replica_scrub from "
-            << msg->map_epoch << " < " << info.history.same_acting_since << dendl;
+    if (finalizing_scrub) {
+      dout(10) << "scrub  pg changed, aborting" << dendl;
+      finalizing_scrub = 0;
+    } else {
+      dout(10) << "replica_scrub discarding old replica_scrub from "
+              << msg->map_epoch << " < " << info.history.same_acting_since 
+              << dendl;
+    }
     msg->put();
     return;
   }
 
   ScrubMap map;
   if (msg->scrub_from > eversion_t()) {
-    epoch_t epoch = info.history.same_acting_since;
-    finalizing_scrub = 1;
-    while (last_update_applied != info.last_update) {
-      wait();
-      if (epoch != info.history.same_acting_since ||
-         osd->is_stopping()) {
-       dout(10) << "scrub  pg changed, aborting" << dendl;
+    if (finalizing_scrub) {
+      assert(last_update_applied == info.last_update);
+    } else {
+      finalizing_scrub = 1;
+      if (last_update_applied != info.last_update) {
+       active_rep_scrub = msg;
        return;
       }
     }
@@ -3108,8 +3125,8 @@ void PG::replica_scrub(MOSDRepScrub *msg)
   }
 
   if (msg->map_epoch < info.history.same_acting_since) {
-    dout(10) << "replica_scrub discarding old replica_scrub result from "
-            << msg->map_epoch << " < " << info.history.same_acting_since << dendl;
+    dout(10) << "scrub  pg changed, aborting" << dendl;
+    msg->put();
     return;
   }
 

diff --git a/src/osd/PG.h b/src/osd/PG.h
index e4b0f1091760257521bc8ede80babc28026a84d7..4842c590dc8dec6a5414cbd28fe7c47355aa9f13 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -24,6 +24,7 @@
 #include "OSDMap.h"
 #include "os/ObjectStore.h"
 #include "msg/Messenger.h"
+#include "messages/MOSDRepScrub.h"
 
 #include "common/DecayCounter.h"
 
@@ -902,6 +903,7 @@ public:
   int scrub_waiting_on;
   epoch_t scrub_epoch_start;
   ScrubMap primary_scrubmap;
+  MOSDRepScrub *active_rep_scrub;
 
   void repair_object(const sobject_t& soid, ScrubMap::object *po, int bad_peer, int ok_peer);
   void scrub();
@@ -944,7 +946,8 @@ public:
     finish_sync_event(NULL),
     finalizing_scrub(false),
     scrub_reserved(false), scrub_reserve_failed(false),
-    scrub_waiting_on(0)
+    scrub_waiting_on(0),
+    active_rep_scrub(0)
   {
     pool->get();
   }

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 932e3dc3d11e9c1b5820313932782d72c6beae44..ccafbfab89bdaae04b81493b9f3bc73af427ed2e 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -3007,7 +3007,10 @@ void ReplicatedPG::sub_op_modify_applied(RepModify *rm)
 
   last_update_applied = rm->op->version;
   if (last_update_applied == info.last_update && finalizing_scrub) {
-    kick();
+    assert(active_rep_scrub);
+    osd->rep_scrub_wq.queue(active_rep_scrub);
+    active_rep_scrub->put();
+    active_rep_scrub = 0;
   }
 
   unlock();