From 048f8096265dd3a647adb970255e4b11c9617b2e Mon Sep 17 00:00:00 2001
From: David Zafman <dzafman@redhat.com>
Date: Mon, 22 Jul 2019 11:52:41 -0700
Subject: [PATCH] osd mgr: Add osd_mon_heartbeat_stat_stale option to time out
 ping info after 1 hour

Signed-off-by: David Zafman <dzafman@redhat.com>
---
 doc/rados/configuration/mon-osd-interaction.rst |  9 +++++++++
 src/common/options.cc                           |  5 +++++
 src/mgr/ClusterState.cc                         | 10 ++++++++++
 src/osd/OSD.cc                                  | 16 ++++++++++++++++
 4 files changed, 40 insertions(+)

diff --git a/doc/rados/configuration/mon-osd-interaction.rst b/doc/rados/configuration/mon-osd-interaction.rst
index 42be922fec0b5..a7324ebb0e554 100644
--- a/doc/rados/configuration/mon-osd-interaction.rst
+++ b/doc/rados/configuration/mon-osd-interaction.rst
@@ -377,6 +377,15 @@ OSD Settings
 :Default: ``30``
 
 
+``osd mon heartbeat stat stale``
+
+:Description: Stop reporting on heartbeat ping times which haven't been updated for
+              this many seconds.  Set to zero to disable this action.
+
+:Type: 32-bit Integer
+:Default: ``3600``
+
+
 ``osd mon report interval``
 
 :Description: The number of seconds a Ceph OSD Daemon may wait
diff --git a/src/common/options.cc b/src/common/options.cc
index 15d88ced4a4bb..9f9d1fbe5ae5d 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3373,6 +3373,11 @@ std::vector<Option> get_global_options() {
     .set_default(30)
     .set_description(""),
 
+    Option("osd_mon_heartbeat_stat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(1_hr)
+    .set_description("Stop reporting on heartbeat ping times not updated for this many seconds.")
+    .set_long_description("Stop reporting on old heartbeat information unless this is set to zero"),
+
     Option("osd_mon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(5)
     .set_description("Frequency of OSD reports to mon for peer failures, fullness status changes"),
diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index a4881f1cb4c40..0bb06229bd90f 100644
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -262,8 +262,18 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
     };
 
     set<mgr_ping_time_t> sorted;
+    utime_t now = ceph_clock_now();
     for (auto i : pg_map.osd_stat) {
       for (auto j : i.second.hb_pingtime) {
+
+	if (j.second.last_update == 0)
+	  continue;
+	auto stale_time = g_ceph_context->_conf.get_val<int64_t>("osd_mon_heartbeat_stat_stale");
+	if (now.sec() - j.second.last_update > stale_time) {
+	  dout(20) << __func__ << " time out heartbeat for osd " << i.first
+	           << " last_update " << j.second.last_update << dendl;
+	   continue;
+	}
 	mgr_ping_time_t item;
 	item.pingtime = std::max(j.second.back_pingtime[0], j.second.back_pingtime[1]);
 	item.pingtime = std::max(item.pingtime, j.second.back_pingtime[2]);
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 496824f330837..0727e821986d6 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -944,10 +944,24 @@ void OSDService::set_statfs(const struct store_statfs_t &stbuf,
 osd_stat_t OSDService::set_osd_stat(vector<int>& hb_peers,
 				    int num_pgs)
 {
+  utime_t now = ceph_clock_now();
+  auto stale_time = g_conf().get_val<int64_t>("osd_mon_heartbeat_stat_stale");
   std::lock_guard l(stat_lock);
   osd_stat.hb_peers.swap(hb_peers);
   osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
   osd_stat.num_pgs = num_pgs;
+  // Clean entries that aren't updated
+  // This is called often enough that we can just remove 1 at a time
+  for (auto i: osd_stat.hb_pingtime) {
+    if (i.second.last_update == 0)
+      continue;
+    if (stale_time && now.sec() - i.second.last_update > stale_time) {
+      dout(20) << __func__ << " time out heartbeat for osd " << i.first
+	       << " last_update " << i.second.last_update << dendl;
+      osd_stat.hb_pingtime.erase(i.first);
+      break;
+    }
+  }
   return osd_stat;
 }
 
@@ -2590,6 +2604,8 @@ will start to track new ops received afterwards.";
     map<int, osd_stat_t::Interfaces> *pingtimes = new map<int, osd_stat_t::Interfaces>;
     service.get_hb_pingtime(pingtimes);
     for (auto j : *pingtimes) {
+      if (j.second.last_update == 0)
+	continue;
       osd_ping_time_t item;
       item.pingtime = std::max(j.second.back_pingtime[0], j.second.back_pingtime[1]);
       item.pingtime = std::max(item.pingtime, j.second.back_pingtime[2]);
-- 
2.39.5