mon: check failures in tick

Currently we only trigger a failure on receipt of a failure report.  Move
the checks into a helper and check during tick() too, so that we will
trigger failures even when the thresholds are not met at failure report
time.  This is rarely true now, but will be true once we locally scale the
grace period.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index d7c99a9f9068748e21e98c5b4055d96fee0a3a66..1f8f5993b1eed6d8ee02fecb78a1b6fdea2e2174 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -673,6 +673,47 @@ bool OSDMonitor::can_mark_in(int i)
   return true;
 }
 
+void OSDMonitor::check_failures(utime_t now)
+{
+  for (map<int,failure_info_t>::iterator p = failure_info.begin();
+       p != failure_info.end();
+       ++p) {
+    check_failure(now, p->first, p->second);
+  }
+}
+
+bool OSDMonitor::check_failure(utime_t now, int target_osd, failure_info_t& fi)
+{
+  utime_t grace(g_conf->osd_heartbeat_grace, 0);
+  utime_t max_failed_since = fi.get_failed_since();
+  utime_t failed_for = now - max_failed_since;
+
+  dout(10) << " osd." << target_osd << " has "
+          << fi.reporters.size() << " reporters and "
+          << fi.num_reports << " reports, "
+          << grace << " grace, max_failed_since " << max_failed_since
+          << dendl;
+
+  if (failed_for >= grace &&
+      ((int)fi.reporters.size() >= g_conf->osd_min_down_reporters) &&
+      (fi.num_reports >= g_conf->osd_min_down_reports)) {
+    dout(1) << " we have enough reports/reporters to mark osd." << target_osd << " down" << dendl;
+    pending_inc.new_state[target_osd] = CEPH_OSD_UP;
+
+    mon->clog.info() << osdmap.get_inst(target_osd) << " failed ("
+                    << fi.num_reports << " reports from " << (int)fi.reporters.size() << " peers after "
+                    << failed_for << ")\n";
+
+    list<MOSDFailure*> msgs;
+    fi.take_report_messages(msgs);
+    failure_info.erase(target_osd);
+
+    paxos->wait_for_commit(new C_Reported(this, msgs));
+    return true;
+  }
+  return false;
+}
+
 bool OSDMonitor::prepare_failure(MOSDFailure *m)
 {
   dout(1) << "prepare_failure " << m->get_target() << " from " << m->get_orig_source_inst()
@@ -697,33 +738,7 @@ bool OSDMonitor::prepare_failure(MOSDFailure *m)
     if (old)
       mon->no_reply(old);
 
-    utime_t grace(g_conf->osd_heartbeat_grace, 0);
-    utime_t max_failed_since = fi.get_failed_since();
-    utime_t failed_for = now - max_failed_since;
-
-    dout(10) << " osd." << target_osd << " has "
-            << fi.reporters.size() << " reporters and "
-            << fi.num_reports << " reports, "
-            << grace << " grace, max_failed_since " << max_failed_since
-            << dendl;
-
-    if (failed_for >= grace &&
-       ((int)fi.reporters.size() >= g_conf->osd_min_down_reporters) &&
-        (fi.num_reports >= g_conf->osd_min_down_reports)) {
-      dout(1) << " we have enough reports/reporters to mark osd." << target_osd << " down" << dendl;
-      pending_inc.new_state[target_osd] = CEPH_OSD_UP;
-
-      mon->clog.info() << m->get_target() << " failed ("
-                      << fi.num_reports << " reports from " << (int)fi.reporters.size() << " peers after "
-                      << failed_for << ")\n";
-
-      list<MOSDFailure*> msgs;
-      fi.take_report_messages(msgs);
-      failure_info.erase(target_osd);
-
-      paxos->wait_for_commit(new C_Reported(this, msgs));
-      return true;
-    }
+    return check_failure(now, target_osd, fi);
   } else {
     // remove the report
     if (failure_info.count(target_osd)) {
@@ -1325,10 +1340,12 @@ void OSDMonitor::tick()
   if (!mon->is_leader()) return;
 
   bool do_propose = false;
-
-  // mark down osds out?
   utime_t now = ceph_clock_now(g_ceph_context);
 
+  // mark osds down?
+  check_failures(now);
+
+  // mark down osds out?
 
   /* can_mark_out() checks if we can mark osds as being out. The -1 has no
    * influence at all. The decision is made based on the ratio of "in" osds,

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 03b2c3444065435dfa259c50388fcb2876f6c5f2..c58636666b705c3d67171d3e7d2a762faaa16f27 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -123,6 +123,9 @@ private:
 
   map<int,double> osd_weight;
 
+  void check_failures(utime_t now);
+  bool check_failure(utime_t now, int target_osd, failure_info_t& fi);
+
   // map thrashing
   int thrash_map;
   int thrash_last_up_osd;