osd mgr: Add osd_mon_heartbeat_stat_stale option to time out ping info

after 1 hour

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 048f8096265dd3a647adb970255e4b11c9617b2e)

Conflicts:
	src/osd/OSD.cc (trivial)

diff --git a/doc/rados/configuration/mon-osd-interaction.rst b/doc/rados/configuration/mon-osd-interaction.rst
index 42be922fec0b51e561feab6a144e25cc7c2c4c9a..a7324ebb0e5541e996e69f8140a3e0ab71e24398 100644 (file)
--- a/doc/rados/configuration/mon-osd-interaction.rst
+++ b/doc/rados/configuration/mon-osd-interaction.rst
@@ -377,6 +377,15 @@ OSD Settings
 :Default: ``30``
 
 
+``osd mon heartbeat stat stale``
+
+:Description: Stop reporting on heartbeat ping times which haven't been updated for
+              this many seconds.  Set to zero to disable this action.
+
+:Type: 32-bit Integer
+:Default: ``3600``
+
+
 ``osd mon report interval``
 
 :Description: The number of seconds a Ceph OSD Daemon may wait

diff --git a/src/common/options.cc b/src/common/options.cc
index ab9d44e56c0204d10503e87c58728235d6e7d301..89af39bc73870b61709b8cad43c40e32be861bf0 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -2969,6 +2969,11 @@ std::vector<Option> get_global_options() {
     .set_default(30)
     .set_description(""),
 
+    Option("osd_mon_heartbeat_stat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(1_hr)
+    .set_description("Stop reporting on heartbeat ping times not updated for this many seconds.")
+    .set_long_description("Stop reporting on old heartbeat information unless this is set to zero"),
+
     Option("osd_mon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(5)
     .set_description("Frequency of OSD reports to mon for peer failures, fullness status changes"),

diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index 1a06881f8e9ec941a0ed5d6f4228f09a156cc52e..40ad33d800109d0a475af7ca81c4e0ba22c5f343 100644 (file)
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -252,8 +252,18 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
     };
 
     set<mgr_ping_time_t> sorted;
+    utime_t now = ceph_clock_now();
     for (auto i : pg_map.osd_stat) {
       for (auto j : i.second.hb_pingtime) {
+
+       if (j.second.last_update == 0)
+         continue;
+       auto stale_time = g_ceph_context->_conf.get_val<int64_t>("osd_mon_heartbeat_stat_stale");
+       if (now.sec() - j.second.last_update > stale_time) {
+         dout(20) << __func__ << " time out heartbeat for osd " << i.first
+                  << " last_update " << j.second.last_update << dendl;
+          continue;
+       }
        mgr_ping_time_t item;
        item.pingtime = std::max(j.second.back_pingtime[0], j.second.back_pingtime[1]);
        item.pingtime = std::max(item.pingtime, j.second.back_pingtime[2]);

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 2af8ea73a425332e443eff741a204f048bd61c49..ca12b25163943657cf80b6c36d133ed4af4e3eb6 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -809,10 +809,24 @@ void OSDService::set_statfs(const struct store_statfs_t &stbuf)
 osd_stat_t OSDService::set_osd_stat(vector<int>& hb_peers,
                                    int num_pgs)
 {
+  utime_t now = ceph_clock_now();
+  auto stale_time = g_conf().get_val<int64_t>("osd_mon_heartbeat_stat_stale");
   Mutex::Locker l(stat_lock);
   osd_stat.hb_peers.swap(hb_peers);
   osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
   osd_stat.num_pgs = num_pgs;
+  // Clean entries that aren't updated
+  // This is called often enough that we can just remove 1 at a time
+  for (auto i: osd_stat.hb_pingtime) {
+    if (i.second.last_update == 0)
+      continue;
+    if (stale_time && now.sec() - i.second.last_update > stale_time) {
+      dout(20) << __func__ << " time out heartbeat for osd " << i.first
+              << " last_update " << i.second.last_update << dendl;
+      osd_stat.hb_pingtime.erase(i.first);
+      break;
+    }
+  }
   return osd_stat;
 }
 
@@ -2323,6 +2337,8 @@ will start to track new ops received afterwards.";
     map<int, osd_stat_t::Interfaces> *pingtimes = new map<int, osd_stat_t::Interfaces>;
     service.get_hb_pingtime(pingtimes);
     for (auto j : *pingtimes) {
+      if (j.second.last_update == 0)
+       continue;
       osd_ping_time_t item;
       item.pingtime = std::max(j.second.back_pingtime[0], j.second.back_pingtime[1]);
       item.pingtime = std::max(item.pingtime, j.second.back_pingtime[2]);