mon: blacklist failed mds

diff --git a/src/TODO b/src/TODO
index c01626bd289bf4f0b2cf21417e2a527f135fea8e..e7273c714e06f35573aa9168f3f674826e5e834b 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -130,7 +130,6 @@ mds
 - anchor_destroy needs to xlock linklock.. which means it needs a Mutation wrapper?
   - ... when it gets a caller.. someday..
 - make truncate faster with a trunc_seq, attached to objects as attributes?
-- osd needs a set_floor_and_read op for safe failover/STOGITH-like semantics.
 - could mark dir complete in EMetaBlob by counting how many dentries are dirtied in the current log epoch in CDir...
 - FIXME how to journal/store root and stray inode content? 
   - in particular, i care about dirfragtree.. get it on rejoin?
@@ -151,7 +150,6 @@ mon
 - paxos need to clean up old states.
   - default: simple max of (state count, min age), so that we have at least N hours of history, say?
   - osd map: trim only old maps < oldest "in" osd up_from
-- blacklist failed mds's
 
 osdmon
 - monitor needs to monitor some osds...

diff --git a/src/config.cc b/src/config.cc
index 8ab996a3edace999b93f2c8b1e5844f8c94d427a..e7437a5a70e57b4e789e92c10fe1c67cee01d561 100644 (file)
--- a/src/config.cc
+++ b/src/config.cc
@@ -329,6 +329,7 @@ md_config_t g_conf = {
 
   mds_beacon_interval: 4, //30.0,
   mds_beacon_grace: 30, //60*60.0,
+  mds_blacklist_interval: 24.0*60.0,  // how long to blacklist failed nodes
 
   mds_session_timeout: 60,    // cap bits and leases time out if client idle
   mds_session_autoclose: 300, // autoclose idle session 
@@ -337,7 +338,6 @@ md_config_t g_conf = {
                               //  make it (mds_session_timeout - mds_beacon_grace)
 
   mds_tick_interval: 5,
-
   mds_scatter_nudge_interval: 5,  // how quickly dirstat changes propagate up the hierarchy
 
   mds_log: true,
@@ -421,7 +421,7 @@ md_config_t g_conf = {
   osd_age_time: 0,
   osd_heartbeat_interval: 1,
   osd_mon_heartbeat_interval: 30,  // if no peers, ping monitor
-  osd_heartbeat_grace: 60,
+  osd_heartbeat_grace: 20,
   osd_mon_report_interval:  5,  // pg stats, failures, up_thru, boot.
   osd_replay_window: 45,
   osd_max_pull: 2,

diff --git a/src/config.h b/src/config.h
index 992a44afced1b40e2e9973ed578eabc4287d6b9f..53f1b9b4920fff837d1733e43bc8642a6cc82c32 100644 (file)
--- a/src/config.h
+++ b/src/config.h
@@ -199,6 +199,7 @@ struct md_config_t {
 
   float mds_beacon_interval;
   float mds_beacon_grace;
+  float mds_blacklist_interval;
 
   float mds_session_timeout;
   float mds_session_autoclose;

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 157aa9c3b1166a6438d390032c1493a8c2851254..5c3320e17cec0b0423c41e21a249fb2b21fe1071 100644 (file)
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -658,7 +658,8 @@ void MDSMonitor::tick()
   }
 
   // check beacon timestamps
-  utime_t cutoff = g_clock.now();
+  utime_t now = g_clock.now();
+  utime_t cutoff = now;
   cutoff -= g_conf.mds_beacon_grace;
 
   // make sure last_beacon is populated
@@ -676,86 +677,95 @@ void MDSMonitor::tick()
     if (last_beacon.count(p->first) == 0)
       last_beacon[p->first] = g_clock.now();
 
-  map<entity_addr_t, utime_t>::iterator p = last_beacon.begin();
-  while (p != last_beacon.end()) {
-    entity_addr_t addr = p->first;
-    utime_t since = p->second;
-    p++;
 
-    if (last_beacon[addr] >= cutoff)
-      continue;
-
-    int mds = pending_mdsmap.get_addr_rank(addr);
-
-    if ((mds < 0 || pending_mdsmap.standby_for.count(mds) == 0) &&
-       pending_mdsmap.standby_any.empty()) {
-      // laggy!
-      dout(10) << "no beacon from mds" << mds << " " << addr << " since " << since
-              << ", marking laggy" << dendl;
-      pending_mdsmap.laggy.insert(addr);
-      do_propose = true;
-      continue;
-    }
-    
-    if (mds >= 0) {
-      // failure!
-      int curstate = pending_mdsmap.get_state(mds);
-      int newstate = curstate;
-      switch (curstate) {
-      case MDSMap::STATE_CREATING:
-      case MDSMap::STATE_DNE:
-       newstate = MDSMap::STATE_DNE;   // didn't finish creating
-       last_beacon.erase(addr);
-       break;
-       
-      case MDSMap::STATE_STARTING:
-       newstate = MDSMap::STATE_STOPPED;
-       break;
-       
-      case MDSMap::STATE_STOPPED:
+  if (mon->osdmon()->paxos->is_writeable()) {
+    map<entity_addr_t, utime_t>::iterator p = last_beacon.begin();
+    while (p != last_beacon.end()) {
+      entity_addr_t addr = p->first;
+      utime_t since = p->second;
+      p++;
+      
+      if (last_beacon[addr] >= cutoff)
+       continue;
+      
+      int mds = pending_mdsmap.get_addr_rank(addr);
+      
+      if ((mds < 0 || pending_mdsmap.standby_for.count(mds) == 0) &&
+         pending_mdsmap.standby_any.empty()) {
+       // laggy!
+       dout(10) << "no beacon from mds" << mds << " " << addr << " since " << since
+                << ", marking laggy" << dendl;
+       pending_mdsmap.laggy.insert(addr);
+       do_propose = true;
+       continue;
+      }
+      
+      if (mds >= 0) {
+       // failure!
+       int curstate = pending_mdsmap.get_state(mds);
+       int newstate = curstate;
+       switch (curstate) {
+       case MDSMap::STATE_CREATING:
+       case MDSMap::STATE_DNE:
+         newstate = MDSMap::STATE_DNE; // didn't finish creating
+         last_beacon.erase(addr);
+         break;
+         
+       case MDSMap::STATE_STARTING:
+         newstate = MDSMap::STATE_STOPPED;
+         break;
+         
+       case MDSMap::STATE_STOPPED:
        break;
        
-      case MDSMap::STATE_REPLAY:
-      case MDSMap::STATE_RESOLVE:
-      case MDSMap::STATE_RECONNECT:
-      case MDSMap::STATE_REJOIN:
-      case MDSMap::STATE_ACTIVE:
-      case MDSMap::STATE_STOPPING:
-      case MDSMap::STATE_FAILED:
-       newstate = MDSMap::STATE_FAILED;
-       pending_mdsmap.last_failure = pending_mdsmap.epoch;
-       break;
+       case MDSMap::STATE_REPLAY:
+       case MDSMap::STATE_RESOLVE:
+       case MDSMap::STATE_RECONNECT:
+       case MDSMap::STATE_REJOIN:
+       case MDSMap::STATE_ACTIVE:
+       case MDSMap::STATE_STOPPING:
+       case MDSMap::STATE_FAILED:
+         newstate = MDSMap::STATE_FAILED;
+         pending_mdsmap.last_failure = pending_mdsmap.epoch;
+         break;
+         
+       default:
+         assert(0);
+       }
        
-      default:
-       assert(0);
+       dout(10) << "no beacon from mds" << mds << " " << addr << " since " << since
+                << ", marking " << pending_mdsmap.get_state_name(newstate)
+                << dendl;
+
+       // blacklist
+       utime_t until = now;
+       until += g_conf.mds_blacklist_interval;
+       mon->osdmon()->blacklist(addr, until);
+       mon->osdmon()->propose_pending();
+
+       // update map
+       pending_mdsmap.mds_state[mds] = newstate;
+       pending_mdsmap.mds_state_seq.erase(mds);
+       pending_mdsmap.laggy.erase(addr);
+      } 
+      else if (pending_mdsmap.is_standby(addr)) {
+       dout(10) << "no beacon from standby " << addr << " since " << last_beacon[addr]
+                << ", removing from standby list"
+                << dendl;
+       if (pending_mdsmap.standby[addr].mds >= 0)
+         pending_mdsmap.standby_for[pending_mdsmap.standby[addr].mds].erase(addr);
+       else
+         pending_mdsmap.standby_any.erase(addr);
+       pending_mdsmap.standby.erase(addr);
+       pending_mdsmap.laggy.erase(addr);
+      } 
+      else {
+       dout(0) << "BUG: removing stray " << addr << " from last_beacon map" << dendl;
       }
       
-      dout(10) << "no beacon from mds" << mds << " " << addr << " since " << since
-              << ", marking " << pending_mdsmap.get_state_name(newstate)
-              << dendl;
-      
-      // update map
-      pending_mdsmap.mds_state[mds] = newstate;
-      pending_mdsmap.mds_state_seq.erase(mds);
-      pending_mdsmap.laggy.erase(addr);
-    } 
-    else if (pending_mdsmap.is_standby(addr)) {
-      dout(10) << "no beacon from standby " << addr << " since " << last_beacon[addr]
-              << ", removing from standby list"
-              << dendl;
-      if (pending_mdsmap.standby[addr].mds >= 0)
-       pending_mdsmap.standby_for[pending_mdsmap.standby[addr].mds].erase(addr);
-      else
-       pending_mdsmap.standby_any.erase(addr);
-      pending_mdsmap.standby.erase(addr);
-      pending_mdsmap.laggy.erase(addr);
-    } 
-    else {
-      dout(0) << "BUG: removing stray " << addr << " from last_beacon map" << dendl;
+      last_beacon.erase(addr);
+      do_propose = true;
     }
-
-    last_beacon.erase(addr);
-    do_propose = true;
   }
 
   if (do_propose)