osd: use distinct wait list for scrub

We've traditionally (ab)used waiting_for_active for ops blocked
by scrub.  In order to preserve the op requeue ordering for
backoffs relating to unreadable and degraded, however, we need
to order the scrub waits different.  The first step is to have it
on its own list.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index c373ef1ac20e376acc2d69bdeb3eaebb2052417a..0b2ddf10d853d2d9bb742f4f5f318b4558344e01 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2215,6 +2215,7 @@ void PG::split_ops(PG *child, unsigned split_bits) {
   assert(waiting_for_degraded_object.empty());
   assert(waiting_for_ondisk.empty());
   assert(waiting_for_active.empty());
+  assert(waiting_for_scrub.empty());
 
   osd->dequeue_pg(this, &waiting_for_peered);
 
@@ -4311,7 +4312,7 @@ void PG::chunky_scrub(ThreadPool::TPHandle &handle)
        scrubber.run_callbacks();
 
         // requeue the writes from the chunk that just finished
-        requeue_ops(waiting_for_active);
+        requeue_ops(waiting_for_scrub);
 
        scrubber.state = PG::Scrubber::WAIT_DIGEST_UPDATES;
 
@@ -4361,7 +4362,7 @@ void PG::scrub_clear_state()
   if (scrubber.active)
     osd->dec_scrubs_active();
 
-  requeue_ops(waiting_for_active);
+  requeue_ops(waiting_for_scrub);
 
   if (scrubber.queue_snap_trim) {
     dout(10) << "scrub finished, requeuing snap_trimmer" << dendl;

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 607fadad7039016b3c0722ec521ea142190c6668..260aa3cf2a7e49ab3dc1f0ed873a8b14a6cbb28e 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -831,6 +831,7 @@ protected:
 
   // ops waiting on active (require peered as well)
   list<OpRequestRef>            waiting_for_active;
+  list<OpRequestRef>            waiting_for_scrub;
 
   list<OpRequestRef>            waiting_for_cache_not_full;
   list<OpRequestRef>            waiting_for_all_missing;

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index cfd88d4b5684f09faac0b96fcecf22e0089ad9cf..89f4446e64db99bacc81738b49a423027dca63e7 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -1858,7 +1858,7 @@ void PrimaryLogPG::do_op(OpRequestRef& op)
   if (write_ordered &&
       scrubber.write_blocked_by_scrub(head, get_sort_bitwise())) {
     dout(20) << __func__ << ": waiting for scrub" << dendl;
-    waiting_for_active.push_back(op);
+    waiting_for_scrub.push_back(op);
     op->mark_delayed("waiting for scrub");
     return;
   }
@@ -2880,10 +2880,10 @@ void PrimaryLogPG::promote_object(ObjectContextRef obc,
     dout(10) << __func__ << " " << hoid
             << " blocked by scrub" << dendl;
     if (op) {
-      waiting_for_active.push_back(op);
+      waiting_for_scrub.push_back(op);
       op->mark_delayed("waiting for scrub");
       dout(10) << __func__ << " " << hoid
-              << " placing op in waiting_for_active" << dendl;
+              << " placing op in waiting_for_scrub" << dendl;
     } else {
       dout(10) << __func__ << " " << hoid
               << " no op, dropping on the floor" << dendl;
@@ -10005,10 +10005,11 @@ void PrimaryLogPG::on_change(ObjectStore::Transaction *t)
   // requeue everything in the reverse order they should be
   // reexamined.
   requeue_ops(waiting_for_peered);
+  requeue_ops(waiting_for_active);
 
   clear_scrub_reserved();
 
-  // requeues waiting_for_active
+  // requeues waiting_for_scrub
   scrub_clear_state();
 
   cancel_copy_ops(is_primary());
@@ -12344,6 +12345,7 @@ bool PrimaryLogPG::agent_choose_mode(bool restart, OpRequestRef op)
       if (op)
        requeue_op(op);
       requeue_ops(waiting_for_active);
+      requeue_ops(waiting_for_scrub);
       requeue_ops(waiting_for_cache_not_full);
       objects_blocked_on_cache_full.clear();
       requeued = true;

diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index fa9af4223fff5cb2e11d577ffd924aba26818fa9..8e3f19c64105ec6ed6e2f8d09d9b7b168056a0aa 100644 (file)
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -824,8 +824,8 @@ protected:
        if (scrubber.write_blocked_by_scrub(
              p.first.get_head(),
              get_sort_bitwise())) {
-         waiting_for_active.splice(
-           waiting_for_active.begin(),
+         waiting_for_scrub.splice(
+           waiting_for_scrub.begin(),
            p.second,
            p.second.begin(),
            p.second.end());