osd: send forced scrub/repair through scrub scheduling

This marks a PG for immediate scrub or repair.  Adjust the sched_scrub()
code so that we handle these PGs even when should_schedule_scrub is
false (e.g., because the load is high).  When we explicitly request a
scrub or repair, we then go through the normal scrub reservation process
to avoid unduly impacting cluster performance.

This is particularly helpful on argonaut, where the final scrub
finalization step blocks writes to the PG, and overlapping scrubs can
exacerbate the problem.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index f2801534c25bddf556900156d79f5200e6e6c9b0..4caaf46638b291b88b70bca4b110ecfe0c29a103 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1949,9 +1949,7 @@ void OSD::tick()
   // periodically kick recovery work queue
   recovery_tp.wake();
   
-  if (scrub_should_schedule()) {
-    sched_scrub();
-  }
+  sched_scrub();
 
   map_lock.get_read();
 
@@ -3090,11 +3088,11 @@ void OSD::handle_scrub(MOSDScrub *m)
       PG *pg = p->second;
       pg->lock();
       if (pg->is_primary()) {
-       if (m->repair)
-         pg->state_set(PG_STATE_REPAIR);
-       if (pg->queue_scrub()) {
-         dout(10) << "queueing " << *pg << " for scrub" << dendl;
-       }
+       pg->unreg_scrub();
+       pg->must_scrub = true;
+       pg->must_repair = m->repair;
+       pg->reg_scrub();
+       dout(10) << "marking " << *pg << " for scrub" << dendl;
       }
       pg->unlock();
     }
@@ -3106,11 +3104,11 @@ void OSD::handle_scrub(MOSDScrub *m)
        PG *pg = pg_map[*p];
        pg->lock();
        if (pg->is_primary()) {
-         if (m->repair)
-           pg->state_set(PG_STATE_REPAIR);
-         if (pg->queue_scrub()) {
-           dout(10) << "queueing " << *pg << " for scrub" << dendl;
-         }
+         pg->unreg_scrub();
+         pg->must_scrub = true;
+         pg->must_repair = m->repair;
+         pg->reg_scrub();
+         dout(10) << "marking " << *pg << " for scrub" << dendl;
        }
        pg->unlock();
       }
@@ -3157,7 +3155,9 @@ void OSD::sched_scrub()
 {
   assert(osd_lock.is_locked());
 
-  dout(20) << "sched_scrub" << dendl;
+  bool should = scrub_should_schedule();
+
+  dout(20) << "sched_scrub should=" << (int)should << dendl;
 
   pair<utime_t,pg_t> pos;
   utime_t max = ceph_clock_now(g_ceph_context);
@@ -3184,7 +3184,9 @@ void OSD::sched_scrub()
     sched_scrub_lock.Unlock();
     PG *pg = _lookup_lock_pg(pgid);
     if (pg) {
-      if (pg->is_active() && !pg->sched_scrub()) {
+      if (pg->is_active() &&
+         (should || pg->must_scrub) &&
+         !pg->sched_scrub()) {
        pg->unlock();
        sched_scrub_lock.Lock();
        break;

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 28293b3c1e39fca7e5158adef9d3f7cdff67ea4e..c9d2a65fa456b18ecf5f72111c15275d1f5ff0dc 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1584,6 +1584,7 @@ bool PG::queue_scrub()
   if (is_scrubbing()) {
     return false;
   }
+  must_scrub = false;
   state_set(PG_STATE_SCRUBBING);
   osd->scrub_wq.queue(this);
   return true;
@@ -2620,7 +2621,11 @@ bool PG::sched_scrub()
 
 void PG::reg_scrub()
 {
+  if (must_scrub) {
+    scrub_reg_stamp = utime_t();
+  } else {
     scrub_reg_stamp = info.history.last_scrub_stamp;
+  }
   osd->reg_last_pg_scrub(info.pgid, scrub_reg_stamp);
 }
 
@@ -3037,7 +3042,6 @@ void PG::scrub()
 
   if (!is_primary() || !is_active() || !is_clean() || !is_scrubbing()) {
     dout(10) << "scrub -- not primary or active or not clean" << dendl;
-    state_clear(PG_STATE_REPAIR);
     state_clear(PG_STATE_SCRUBBING);
     clear_scrub_reserved();
     unlock();
@@ -3145,7 +3149,6 @@ void PG::scrub_clear_state()
 {
   assert(_lock.is_locked());
   state_clear(PG_STATE_SCRUBBING);
-  state_clear(PG_STATE_REPAIR);
   update_stats();
 
   // active -> nothing.
@@ -3153,6 +3156,9 @@ void PG::scrub_clear_state()
 
   osd->requeue_ops(this, waiting_for_active);
 
+  must_scrub = false;
+  must_repair = false;
+
   finalizing_scrub = false;
   scrub_block_writes = false;
   scrub_active = false;
@@ -3318,7 +3324,7 @@ void PG::scrub_finalize() {
 
   dout(10) << "scrub_finalize has maps, analyzing" << dendl;
   int errors = 0, fixed = 0;
-  bool repair = state_test(PG_STATE_REPAIR);
+  bool repair = must_repair;
   const char *mode = repair ? "repair":"scrub";
   if (acting.size() > 1) {
     dout(10) << "scrub  comparing replica scrub maps" << dendl;
@@ -3716,6 +3722,9 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
   state_clear(PG_STATE_DOWN);
   state_clear(PG_STATE_RECOVERING);
 
+  must_scrub = false;
+  must_repair = false;
+
   peer_missing.clear();
   peer_purged.clear();
 

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 3c4353dab51c7b22e9fb336269872faea7a36d0a..1c680bfea2de57f741141febbd30a8d33b81be27 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -747,6 +747,7 @@ public:
   epoch_t scrub_epoch_start;
   ScrubMap primary_scrubmap;
   MOSDRepScrub *active_rep_scrub;
+  bool must_scrub, must_repair;
   utime_t scrub_reg_stamp;
 
   void repair_object(const hobject_t& soid, ScrubMap::object *po, int bad_peer, int ok_peer);
@@ -1269,6 +1270,7 @@ public:
     scrub_reserved(false), scrub_reserve_failed(false),
     scrub_waiting_on(0),
     active_rep_scrub(0),
+    must_scrub(false), must_repair(false),
     recovery_state(this)
   {
     pool->get();

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index d6ff0d2917f25a657c205a8d53dd33523cb42d37..92df99778c6c718e6c00a339397d7133d001006a 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -5738,7 +5738,8 @@ void ReplicatedPG::on_change()
     scrub_clear_state();
   } else if (is_scrubbing()) {
     state_clear(PG_STATE_SCRUBBING);
-    state_clear(PG_STATE_REPAIR);
+    must_scrub = false;
+    must_repair = false;
   }
 
   context_registry_on_change();
@@ -6461,7 +6462,7 @@ int ReplicatedPG::_scrub(ScrubMap& scrubmap, int& errors, int& fixed)
   dout(10) << "_scrub" << dendl;
 
   coll_t c(info.pgid);
-  bool repair = state_test(PG_STATE_REPAIR);
+  bool repair = must_repair;
   const char *mode = repair ? "repair":"scrub";
 
   // traverse in reverse order.