osd: scrub least recently scrubbed pgs first; once a day

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/config.cc b/src/config.cc
index 34bbc2b784dcf1070368b1420fa71d10404a9c00..592f082858c751ef3e0c8e00a3acb2e8a8e446f9 100644 (file)
--- a/src/config.cc
+++ b/src/config.cc
@@ -540,7 +540,7 @@ static struct config_option config_optionsp[] = {
        OPTION(osd_max_scrubs, 0, OPT_INT, 1),
        OPTION(osd_scrub_load_threshold, 0, OPT_FLOAT, 0.5),
        OPTION(osd_scrub_min_interval, 0, OPT_FLOAT, 300),
-       OPTION(osd_scrub_max_interval, 0, OPT_FLOAT, 60*60*24*7*4),   // 4 weeks
+       OPTION(osd_scrub_max_interval, 0, OPT_FLOAT, 60*60*24),   // once a day
        OPTION(osd_auto_weight, 0, OPT_BOOL, false),
        OPTION(osd_class_error_timeout, 0, OPT_DOUBLE, 60.0),  // seconds
        OPTION(osd_class_timeout, 0, OPT_DOUBLE, 60*60.0), // seconds

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 866978a8e083684dd037f791613fe1422aca7609..35a5f8532de2288b20bd176fc2ae748717cd8a5c 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2261,38 +2261,49 @@ void OSD::sched_scrub()
 
   dout(20) << "sched_scrub" << dendl;
 
+  pair<utime_t,pg_t> pos;
+  utime_t max = g_clock.now();
+  max -= g_conf.osd_scrub_max_interval;
+  
   sched_scrub_lock.Lock();
-  hash_map<pg_t, PG*>::iterator p = pg_map.find(sched_scrub_pg);
-  sched_scrub_lock.Unlock();
 
-  if (p == pg_map.end()) {
-    dout(10) << "starting sched_scrub_pg at the beginning" << dendl;
-    p = pg_map.begin();
-  }
+  //dout(20) << " " << last_scrub_pg << dendl;
 
-  while (p != pg_map.end()) {
-    PG *pg = p->second;
+  set< pair<utime_t,pg_t> >::iterator p = last_scrub_pg.begin();
+  while (p != last_scrub_pg.end()) {
+    //dout(10) << "pos is " << *p << dendl;
+    pos = *p;
+    utime_t t = pos.first;
+    pg_t pgid = pos.second;
 
-    pg->lock();
-    if (!pg->sched_scrub()) {
-      // continue waiting
-      pg->unlock();
+    if (t > max) {
+      dout(10) << " " << pgid << " at " << t
+              << " > " << max << " (" << g_conf.osd_scrub_max_interval << " seconds ago)" << dendl;
       break;
     }
-    pg->unlock();
-    ++p;
-  }
-
-  if (p == pg_map.end()) {
-    p = pg_map.begin();
-  }
 
-  // no pgs!?
-  sched_scrub_lock.Lock();
-  if (p != pg_map.end()) {
-    sched_scrub_pg = p->first;
-  }
+    dout(10) << " on " << t << " " << pgid << dendl;
+    sched_scrub_lock.Unlock();
+    PG *pg = _lookup_lock_pg(pgid);
+    if (pg) {
+      if (pg->is_active() && !pg->sched_scrub()) {
+       pg->unlock();
+       sched_scrub_lock.Lock();
+       break;
+      }
+      pg->unlock();
+    }
+    sched_scrub_lock.Lock();
+
+    // next!
+    p = last_scrub_pg.lower_bound(pos);
+    //dout(10) << "lb is " << *p << dendl;
+    if (p != last_scrub_pg.end())
+      p++;
+  }    
   sched_scrub_lock.Unlock();
+
+  dout(20) << "sched_scrub done" << dendl;
 }
 
 bool OSD::inc_scrubs_pending()
@@ -3679,6 +3690,7 @@ void OSD::handle_pg_notify(MOSDPGNotify *m)
        pg->up.swap(up);
        pg->set_role(role);
        pg->info.history = history;
+       reg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
        pg->clear_primary_state();  // yep, notably, set hml=false
        pg->write_info(*t);
        pg->write_log(*t);
@@ -3714,7 +3726,10 @@ void OSD::handle_pg_notify(MOSDPGNotify *m)
     } else {
       dout(10) << *pg << " got osd" << from << " info " << *it << dendl;
       pg->peer_info[from] = *it;
+
+      unreg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
       pg->info.history.merge(it->history);
+      reg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
 
       // stray?
       if (!pg->is_acting(from)) {
@@ -3783,6 +3798,7 @@ void OSD::_process_pg_info(epoch_t epoch, int from,
     pg->up.swap(up);
     pg->set_role(role);
     pg->info.history = info.history;
+    reg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
     pg->write_info(*t);
     pg->write_log(*t);
     created++;
@@ -3799,7 +3815,9 @@ void OSD::_process_pg_info(epoch_t epoch, int from,
   assert(pg);
 
   dout(10) << *pg << " got " << info << " " << log << " " << missing << dendl;
+  unreg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
   pg->info.history.merge(info.history);
+  reg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
 
   // dump log
   dout(15) << *pg << " my log = ";
@@ -4026,6 +4044,7 @@ void OSD::handle_pg_query(MOSDPGQuery *m)
       pg->up.swap( up );
       pg->set_role(role);
       pg->info.history = history;
+      reg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
       pg->write_info(*t);
       pg->write_log(*t);
       int tr = store->queue_transaction(&pg->osr, t);
@@ -4057,7 +4076,9 @@ void OSD::handle_pg_query(MOSDPGQuery *m)
       continue;
     }
 
+    unreg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
     pg->info.history.merge(it->second.history);
+    reg_last_pg_scrub(pg->info.pgid, pg->info.history.last_scrub_stamp);
 
     // ok, process query!
     assert(!pg->acting.empty());

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 8e267f229a042fc7537abbd1e19690d75605d465..b418dd711aae1d022fb567adc7d7f97c14c0eb03 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -805,12 +805,23 @@ protected:
 
   // -- scrub scheduling --
   Mutex sched_scrub_lock;
-  pg_t sched_scrub_pg;
   int scrubs_pending;
   int scrubs_active;
+  set< pair<utime_t,pg_t> > last_scrub_pg;
+
   bool scrub_should_schedule();
   void sched_scrub();
 
+  void reg_last_pg_scrub(pg_t pgid, utime_t t) {
+    Mutex::Locker l(sched_scrub_lock);
+    last_scrub_pg.insert(pair<utime_t,pg_t>(t, pgid));
+  }
+  void unreg_last_pg_scrub(pg_t pgid, utime_t t) {
+    Mutex::Locker l(sched_scrub_lock);
+    pair<utime_t,pg_t> p(t, pgid);
+    last_scrub_pg.erase(p);
+  }
+
   bool inc_scrubs_pending();
   void dec_scrubs_pending();
   void dec_scrubs_active();

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 034ec705aabd362424e905439870b1e96f0752fa..df4cd9c82d7b6cf5d49b6f860646df8c4e72841d 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -3180,8 +3180,10 @@ void PG::scrub()
   state_clear(PG_STATE_REPAIR);
 
   // finish up
+  osd->unreg_last_pg_scrub(info.pgid, info.history.last_scrub_stamp);
   info.history.last_scrub = info.last_update;
   info.history.last_scrub_stamp = g_clock.now();
+  osd->reg_last_pg_scrub(info.pgid, info.history.last_scrub_stamp);
 
   {
     ObjectStore::Transaction *t = new ObjectStore::Transaction;