From: David Zafman <dzafman@redhat.com>
Date: Mon, 22 Jul 2019 18:52:41 +0000 (-0700)
Subject: osd mgr: Add osd_mon_heartbeat_stat_stale option to time out ping info
X-Git-Tag: v13.2.7~26^2~11
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b3bdc19ec751e3819c9f4698a3566d99997c87ec;p=ceph.git

osd mgr: Add osd_mon_heartbeat_stat_stale option to time out ping info
after 1 hour

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 048f8096265dd3a647adb970255e4b11c9617b2e)

Conflicts:
	src/osd/OSD.cc (trivial)
---

diff --git a/doc/rados/configuration/mon-osd-interaction.rst b/doc/rados/configuration/mon-osd-interaction.rst
index 42be922fec0..a7324ebb0e5 100644
--- a/doc/rados/configuration/mon-osd-interaction.rst
+++ b/doc/rados/configuration/mon-osd-interaction.rst
@@ -377,6 +377,15 @@ OSD Settings
 :Default: ``30``
 
 
+``osd mon heartbeat stat stale``
+
+:Description: Stop reporting on heartbeat ping times which haven't been updated for
+              this many seconds.  Set to zero to disable this action.
+
+:Type: 32-bit Integer
+:Default: ``3600``
+
+
 ``osd mon report interval``
 
 :Description: The number of seconds a Ceph OSD Daemon may wait
diff --git a/src/common/options.cc b/src/common/options.cc
index ab9d44e56c0..89af39bc738 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -2969,6 +2969,11 @@ std::vector<Option> get_global_options() {
     .set_default(30)
     .set_description(""),
 
+    Option("osd_mon_heartbeat_stat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(1_hr)
+    .set_description("Stop reporting on heartbeat ping times not updated for this many seconds.")
+    .set_long_description("Stop reporting on old heartbeat information unless this is set to zero"),
+
     Option("osd_mon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(5)
     .set_description("Frequency of OSD reports to mon for peer failures, fullness status changes"),
diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index 1a06881f8e9..40ad33d8001 100644
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -252,8 +252,18 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
     };
 
     set<mgr_ping_time_t> sorted;
+    utime_t now = ceph_clock_now();
     for (auto i : pg_map.osd_stat) {
       for (auto j : i.second.hb_pingtime) {
+
+	if (j.second.last_update == 0)
+	  continue;
+	auto stale_time = g_ceph_context->_conf.get_val<int64_t>("osd_mon_heartbeat_stat_stale");
+	if (now.sec() - j.second.last_update > stale_time) {
+	  dout(20) << __func__ << " time out heartbeat for osd " << i.first
+	           << " last_update " << j.second.last_update << dendl;
+	   continue;
+	}
 	mgr_ping_time_t item;
 	item.pingtime = std::max(j.second.back_pingtime[0], j.second.back_pingtime[1]);
 	item.pingtime = std::max(item.pingtime, j.second.back_pingtime[2]);
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 2af8ea73a42..ca12b251639 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -809,10 +809,24 @@ void OSDService::set_statfs(const struct store_statfs_t &stbuf)
 osd_stat_t OSDService::set_osd_stat(vector<int>& hb_peers,
 				    int num_pgs)
 {
+  utime_t now = ceph_clock_now();
+  auto stale_time = g_conf().get_val<int64_t>("osd_mon_heartbeat_stat_stale");
   Mutex::Locker l(stat_lock);
   osd_stat.hb_peers.swap(hb_peers);
   osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
   osd_stat.num_pgs = num_pgs;
+  // Clean entries that aren't updated
+  // This is called often enough that we can just remove 1 at a time
+  for (auto i: osd_stat.hb_pingtime) {
+    if (i.second.last_update == 0)
+      continue;
+    if (stale_time && now.sec() - i.second.last_update > stale_time) {
+      dout(20) << __func__ << " time out heartbeat for osd " << i.first
+	       << " last_update " << i.second.last_update << dendl;
+      osd_stat.hb_pingtime.erase(i.first);
+      break;
+    }
+  }
   return osd_stat;
 }
 
@@ -2323,6 +2337,8 @@ will start to track new ops received afterwards.";
     map<int, osd_stat_t::Interfaces> *pingtimes = new map<int, osd_stat_t::Interfaces>;
     service.get_hb_pingtime(pingtimes);
     for (auto j : *pingtimes) {
+      if (j.second.last_update == 0)
+	continue;
       osd_ping_time_t item;
       item.pingtime = std::max(j.second.back_pingtime[0], j.second.back_pingtime[1]);
       item.pingtime = std::max(item.pingtime, j.second.back_pingtime[2]);