mon: Monitor: allow scrub to timeout after a while

Signed-off-by: Joao Eduardo Luis <joao@suse.de>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 5bec749c013e55efc8a7be14554e7498cb0c9323..cbb4d3197f50aa88bd4be9e9a4ff5254ca4d784f 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -244,6 +244,7 @@ OPTION(mon_data_avail_crit, OPT_INT, 5)
 OPTION(mon_data_avail_warn, OPT_INT, 30)
 OPTION(mon_data_size_warn, OPT_U64, 15*1024*1024*1024) // issue a warning when the monitor's data store goes over 15GB (in bytes)
 OPTION(mon_scrub_interval, OPT_INT, 3600*24) // once a day
+OPTION(mon_scrub_timeout, OPT_INT, 60*5) // let's give it 5 minutes; why not.
 OPTION(mon_scrub_max_keys, OPT_INT, 100) // max number of keys to scrub each time
 OPTION(mon_scrub_inject_crc_mismatch, OPT_DOUBLE, 0.0) // probability of injected crc mismatch [0.0, 1.0]
 OPTION(mon_scrub_inject_missing_keys, OPT_DOUBLE, 0.0) // probability of injected missing keys [0.0, 1.0]

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index b21cd38f39afd0b27a823b2cbddc6f283088a405..c16c2c724507ce351efe3e2386fea7c65b132ded 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -168,6 +168,7 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s,
   // scrub
   scrub_version(0),
   scrub_event(NULL),
+  scrub_timeout_event(NULL),
 
   // sync state
   sync_provider_count(0),
@@ -4221,9 +4222,11 @@ int Monitor::scrub()
   assert(is_leader());
   assert(scrub_state);
 
+  scrub_cancel_timeout();
   wait_for_paxos_write();
   scrub_version = paxos->get_version();
 
+
   // scrub all keys if we're the only monitor in the quorum
   int32_t num_keys =
     (quorum.size() == 1 ? -1 : cct->_conf->mon_scrub_max_keys);
@@ -4246,6 +4249,12 @@ int Monitor::scrub()
 
   scrub_state->finished = !r;
 
+  // only after we got our scrub results do we really care whether the
+  // other monitors are late on their results.  Also, this way we avoid
+  // triggering the timeout if we end up getting stuck in _scrub() for
+  // longer than the duration of the timeout.
+  scrub_reset_timeout();
+
   if (quorum.size() == 1) {
     assert(scrub_state->finished == true);
     scrub_finish();
@@ -4283,6 +4292,9 @@ void Monitor::handle_scrub(MMonScrub *m)
        break;
       if (m->version != scrub_version)
        break;
+      // reset the timeout each time we get a result
+      scrub_reset_timeout();
+
       int from = m->get_source().num();
       assert(scrub_result.count(from) == 0);
       scrub_result[from] = m->result;
@@ -4389,6 +4401,13 @@ void Monitor::scrub_check_results()
     clog->info() << "scrub ok on " << quorum << ": " << mine << "\n";
 }
 
+inline void Monitor::scrub_timeout()
+{
+  dout(1) << __func__ << " restarting scrub" << dendl;
+  scrub_reset();
+  scrub_start();
+}
+
 void Monitor::scrub_finish()
 {
   dout(10) << __func__ << dendl;
@@ -4399,6 +4418,7 @@ void Monitor::scrub_finish()
 void Monitor::scrub_reset()
 {
   dout(10) << __func__ << dendl;
+  scrub_cancel_timeout();
   scrub_version = 0;
   scrub_result.clear();
   scrub_state.reset();
@@ -4425,6 +4445,22 @@ void Monitor::scrub_event_cancel()
   }
 }
 
+inline void Monitor::scrub_cancel_timeout()
+{
+  if (scrub_timeout_event) {
+    timer.cancel_event(scrub_timeout_event);
+    scrub_timeout_event = NULL;
+  }
+}
+
+void Monitor::scrub_reset_timeout()
+{
+  dout(15) << __func__ << " reset timeout event" << dendl;
+  scrub_cancel_timeout();
+  scrub_timeout_event = new C_ScrubTimeout(this);
+  timer.add_event_after(g_conf->mon_scrub_timeout, scrub_timeout_event);
+}
+
 /************ TICK ***************/
 
 class C_Mon_Tick : public Context {

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index eff10467087b7c51b935ff237e83b98a5040ba9b..204aabf53c21a02069ef1fc85b2012f1fb086b9a 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -249,6 +249,7 @@ private:
               pair<string,string> *start,
               int *num_keys);
   void scrub_check_results();
+  void scrub_timeout();
   void scrub_finish();
   void scrub_reset();
 
@@ -259,9 +260,19 @@ private:
       mon->scrub_start();
     }
   };
+  struct C_ScrubTimeout : public Context {
+    Monitor *mon;
+    C_ScrubTimeout(Monitor *m) : mon(m) { }
+    void finish(int r) {
+      mon->scrub_timeout();
+    }
+  };
   Context *scrub_event;       ///< periodic event to trigger scrub (leader)
+  Context *scrub_timeout_event;  ///< scrub round timeout (leader)
   void scrub_event_start();
   void scrub_event_cancel();
+  void scrub_reset_timeout();
+  void scrub_cancel_timeout();
 
   struct ScrubState {
     pair<string,string> last_key; ///< last scrubbed key