osd/OSD: auto mark heartbeat sessions as stale and tear them down

The primary benefit is that the OSD doesn't need to keep a flood of
blocked heartbeat messages around in memory.
This prevents OSDs from accumulating heartbeat messages due to a
broken switch and then exhausting the whole node's memory:

Jun 11 04:19:26 host-192-168-9-12 kernel: [409881.137077] Out of memory:
Kill process 1471476 (ceph-osd) score 47 or sacrifice child
Jun 11 04:19:26 host-192-168-9-12 kernel: [409881.146054] Killed process
1471476 (ceph-osd) total-vm:4822548kB, anon-rss:3097860kB,
file-rss:2556kB, shmem-rss:0kB

Fixes: http://tracker.ceph.com/issues/40586
Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
(cherry picked from commit 6cc90f363b8096d2d5fad30e57426d0cea9e3478)

Conflicts:
	src/osd/OSD.cc (no boot_finisher.stop() and no lock_guard)
	src/osd/OSD.h (trivial)

Fixed get_val() call in reset_heartbeat_peers()

diff --git a/src/common/options.cc b/src/common/options.cc
index e52b5d533642b7e5c80fa2d725dd432dd622db96..42c9f73fbddb191a8a1babbf441dc15c314ae1f7 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -2941,6 +2941,13 @@ std::vector<Option> get_global_options() {
     .set_default(20)
     .set_description(""),
 
+    Option("osd_heartbeat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(600)
+    .set_description("Interval (in seconds) we mark an unresponsive heartbeat peer as stale.")
+    .set_long_description("Automatically mark unresponsive heartbeat sessions as stale and tear them down. "
+                         "The primary benefit is that OSD doesn't need to keep a flood of "
+                         "blocked heartbeat messages around in memory."),
+
     Option("osd_heartbeat_min_peers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(10)
     .set_description(""),

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index adf864b2c22b66360e63ac9d2cd301e0a3472ebd..3b87c6a3e4059045b9bf4d98843706a880e80732 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3569,7 +3569,7 @@ int OSD::shutdown()
 
   osd_lock.Lock();
 
-  reset_heartbeat_peers();
+  reset_heartbeat_peers(true);
 
   tick_timer.shutdown();
 
@@ -4572,7 +4572,9 @@ void OSD::maybe_update_heartbeat_peers()
        dout(10) << "maybe_update_heartbeat_peers forcing update after " << dur << " seconds" << dendl;
        heartbeat_set_peers_need_update();
        last_heartbeat_resample = now;
-       reset_heartbeat_peers();   // we want *new* peers!
+       // automatically clean up any stale heartbeat peers
+       // if we are unhealthy, then clean all
+       reset_heartbeat_peers(is_waiting_for_healthy());
       }
     }
   }
@@ -4655,20 +4657,27 @@ void OSD::maybe_update_heartbeat_peers()
   dout(10) << "maybe_update_heartbeat_peers " << heartbeat_peers.size() << " peers, extras " << extras << dendl;
 }
 
-void OSD::reset_heartbeat_peers()
+void OSD::reset_heartbeat_peers(bool all)
 {
   assert(osd_lock.is_locked());
   dout(10) << "reset_heartbeat_peers" << dendl;
+  utime_t stale = ceph_clock_now();
+  stale -= cct->_conf->get_val<int64_t>("osd_heartbeat_stale");
   Mutex::Locker l(heartbeat_lock);
-  while (!heartbeat_peers.empty()) {
-    HeartbeatInfo& hi = heartbeat_peers.begin()->second;
-    hi.con_back->mark_down();
-    if (hi.con_front) {
-      hi.con_front->mark_down();
+  for (auto it = heartbeat_peers.begin(); it != heartbeat_peers.end();) {
+    HeartbeatInfo& hi = it->second;
+    if (all || hi.is_stale(stale)) {
+      hi.con_back->mark_down();
+      if (hi.con_front) {
+        hi.con_front->mark_down();
+      }
+      // stop sending failure_report to mon too
+      failure_queue.erase(it->first);
+      heartbeat_peers.erase(it++);
+    } else {
+      it++;
     }
-    heartbeat_peers.erase(heartbeat_peers.begin());
   }
-  failure_queue.clear();
 }
 
 void OSD::handle_osd_ping(MOSDPing *m)
@@ -8029,7 +8038,7 @@ void OSD::_committed_osd_maps(epoch_t first, epoch_t last, MOSDMap *m)
        hb_front_client_messenger->mark_down_all();
        hb_back_client_messenger->mark_down_all();
 
-       reset_heartbeat_peers();
+       reset_heartbeat_peers(true);
       }
     }
   }

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 4a04d3ba52553b80a3b30389123eb58f740d49b8..06bd201648799210aa60b9cd8af8125edb9be5fa 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1542,6 +1542,14 @@ private:
     vector<uint32_t> hb_front_min;
     vector<uint32_t> hb_front_max;
 
+    bool is_stale(utime_t stale) {
+      if (ping_history.empty()) {
+        return false;
+      }
+      utime_t oldest_deadline = ping_history.begin()->second.first;
+      return oldest_deadline <= stale;
+    }
+
     bool is_unhealthy(utime_t now) {
       if (ping_history.empty()) {
         /// we haven't sent a ping yet or we have got all replies,
@@ -1589,7 +1597,7 @@ private:
   void _remove_heartbeat_peer(int p);
   bool heartbeat_reset(Connection *con);
   void maybe_update_heartbeat_peers();
-  void reset_heartbeat_peers();
+  void reset_heartbeat_peers(bool all);
   bool heartbeat_peers_need_update() {
     return heartbeat_need_update.load();
   }