osd/OSD: auto mark heartbeat sessions as stale and tear them down

The primary benefit is that the OSD doesn't need to keep a flood of
blocked heartbeat messages around in memory.
This prevents OSDs from accumulating heartbeat messages due to a
broken switch and then exhausting the whole node's memory:

Jun 11 04:19:26 host-192-168-9-12 kernel: [409881.137077] Out of memory:
Kill process 1471476 (ceph-osd) score 47 or sacrifice child
Jun 11 04:19:26 host-192-168-9-12 kernel: [409881.146054] Killed process
1471476 (ceph-osd) total-vm:4822548kB, anon-rss:3097860kB,
file-rss:2556kB, shmem-rss:0kB

Fixes: http://tracker.ceph.com/issues/40586
Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
(cherry picked from commit 6cc90f363b8096d2d5fad30e57426d0cea9e3478)

diff --git a/src/common/options.cc b/src/common/options.cc
index 9c79197cf93d63e1902665dec9dc9bc90cf652ad..b3ab211cbf8777f12ee010192ce09e4eb3f7d078 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3323,6 +3323,13 @@ std::vector<Option> get_global_options() {
     .set_default(20)
     .set_description(""),
 
+    Option("osd_heartbeat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(600)
+    .set_description("Interval (in seconds) we mark an unresponsive heartbeat peer as stale.")
+    .set_long_description("Automatically mark unresponsive heartbeat sessions as stale and tear them down. "
+                         "The primary benefit is that OSD doesn't need to keep a flood of "
+                         "blocked heartbeat messages around in memory."),
+
     Option("osd_heartbeat_min_peers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(10)
     .set_description(""),

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index abfdde2f30ba579bcedeae498805a71b3070c6a0..b0136dee5427fb3b9bf73c1742f1a78fc4543782 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3884,7 +3884,7 @@ int OSD::shutdown()
   osd_lock.Lock();
 
   boot_finisher.stop();
-  reset_heartbeat_peers();
+  reset_heartbeat_peers(true);
 
   tick_timer.shutdown();
 
@@ -4857,9 +4857,9 @@ void OSD::maybe_update_heartbeat_peers()
        dout(10) << "maybe_update_heartbeat_peers forcing update after " << dur << " seconds" << dendl;
        heartbeat_set_peers_need_update();
        last_heartbeat_resample = now;
-        if (is_waiting_for_healthy()) {
-         reset_heartbeat_peers();   // we want *new* peers!
-        }
+       // automatically clean up any stale heartbeat peers
+       // if we are unhealthy, then clean all
+       reset_heartbeat_peers(is_waiting_for_healthy());
       }
     }
   }
@@ -4949,20 +4949,27 @@ void OSD::maybe_update_heartbeat_peers()
   dout(10) << "maybe_update_heartbeat_peers " << heartbeat_peers.size() << " peers, extras " << extras << dendl;
 }
 
-void OSD::reset_heartbeat_peers()
+void OSD::reset_heartbeat_peers(bool all)
 {
   ceph_assert(osd_lock.is_locked());
   dout(10) << "reset_heartbeat_peers" << dendl;
+  utime_t stale = ceph_clock_now();
+  stale -= cct->_conf.get_val<int64_t>("osd_heartbeat_stale");
   std::lock_guard l(heartbeat_lock);
-  while (!heartbeat_peers.empty()) {
-    HeartbeatInfo& hi = heartbeat_peers.begin()->second;
-    hi.con_back->mark_down();
-    if (hi.con_front) {
-      hi.con_front->mark_down();
+  for (auto it = heartbeat_peers.begin(); it != heartbeat_peers.end();) {
+    HeartbeatInfo& hi = it->second;
+    if (all || hi.is_stale(stale)) {
+      hi.con_back->mark_down();
+      if (hi.con_front) {
+        hi.con_front->mark_down();
+      }
+      // stop sending failure_report to mon too
+      failure_queue.erase(it->first);
+      heartbeat_peers.erase(it++);
+    } else {
+      it++;
     }
-    heartbeat_peers.erase(heartbeat_peers.begin());
   }
-  failure_queue.clear();
 }
 
 void OSD::handle_osd_ping(MOSDPing *m)
@@ -8369,7 +8376,7 @@ void OSD::_committed_osd_maps(epoch_t first, epoch_t last, MOSDMap *m)
        hb_front_client_messenger->mark_down_all();
        hb_back_client_messenger->mark_down_all();
 
-       reset_heartbeat_peers();
+       reset_heartbeat_peers(true);
       }
     }
   }

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 517a280b83046300933dfb81addbbc86da89d74f..a179f391280713a40274e934df393e69fcde7197 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1563,6 +1563,14 @@ private:
     /// send time -> deadline -> remaining replies
     map<utime_t, pair<utime_t, int>> ping_history;
 
+    bool is_stale(utime_t stale) {
+      if (ping_history.empty()) {
+        return false;
+      }
+      utime_t oldest_deadline = ping_history.begin()->second.first;
+      return oldest_deadline <= stale;
+    }
+
     bool is_unhealthy(utime_t now) {
       if (ping_history.empty()) {
         /// we haven't sent a ping yet or we have got all replies,
@@ -1606,7 +1614,7 @@ private:
   void _remove_heartbeat_peer(int p);
   bool heartbeat_reset(Connection *con);
   void maybe_update_heartbeat_peers();
-  void reset_heartbeat_peers();
+  void reset_heartbeat_peers(bool all);
   bool heartbeat_peers_need_update() {
     return heartbeat_need_update.load();
   }