mon/HealthMonitor: avoid MON_DOWN for freshly added Monitor

In testing, we often have the scenario where cephadm has created a
cluster but doesn't add more monitors until well past
mon_down_mkfs_grace. This causes useless MON_DOWN warnings to be thrown
which fails QA jobs. Avoid this situation entirely by giving a
reasonable grace period for a monitor added to the MonMap to join
quorum.

Fixes: https://tracker.ceph.com/issues/73934
Signed-off-by: Patrick Donnelly <pdonnell@ibm.com>

diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in
index 4f9d9b012ff01e284dbc3b46866ab6fb24f84dce..c52ca49a412e01aeb138d3e403620c3d72e9a428 100644 (file)
--- a/src/common/options/mon.yaml.in
+++ b/src/common/options/mon.yaml.in
@@ -62,6 +62,15 @@ options:
   - mon
   flags:
   - runtime
+- name: mon_down_added_grace
+  type: secs
+  level: advanced
+  desc: Period in seconds that the cluster may have a newly added mon down
+  default: 3_min
+  services:
+  - mon
+  flags:
+  - runtime
 - name: mon_down_uptime_grace
   type: secs
   level: advanced

diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index b5fe1ee9485eeb3118ee418e7c671e425e6f7d21..66faec05c883bf7080863897a7fb724680b4e355 100644 (file)
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -811,30 +811,38 @@ void HealthMonitor::check_for_mon_down(health_check_map_t *checks, std::set<std:
 {
   int max = mon.monmap->size();
   int actual = mon.get_quorum().size();
+  const auto mon_down_mkfs_grace = g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace");
+  const auto mon_down_uptime_grace = g_conf().get_val<std::chrono::seconds>("mon_down_uptime_grace");
+  const auto mon_down_added_grace = g_conf().get_val<std::chrono::seconds>("mon_down_added_grace");
+
   const auto rcnow = ceph::real_clock::now();
   const auto created = mon.monmap->created.to_real_time();
   const auto mcnow = ceph::coarse_mono_clock::now();
   const auto starttime = mon.get_starttime();
 
-  if (actual < max &&
-      (rcnow - created) > g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace") &&
-      (mcnow - starttime) > g_conf().get_val<std::chrono::seconds>("mon_down_uptime_grace")) {
-    ostringstream ss;
-    ss << (max-actual) << "/" << max << " mons down, quorum "
-       << mon.get_quorum_names();
-    auto& d = checks->add("MON_DOWN", HEALTH_WARN, ss.str(), max - actual);
-    set<int> q = mon.get_quorum();
+  if (actual < max && ((rcnow - created) > mon_down_mkfs_grace) && ((mcnow - starttime) > mon_down_uptime_grace)) {
+    auto q = mon.get_quorum();
+    std::list<std::string> details;
     for (int i=0; i<max; i++) {
       if (q.count(i) == 0) {
-       ostringstream ss;
-  std::string mon_name = mon.monmap->get_name(i);
-  mon_downs.insert(mon_name);
-       ss << "mon." << mon_name << " (rank " << i
-          << ") addr " << mon.monmap->get_addrs(i)
-          << " is down (out of quorum)";
-       d.detail.push_back(ss.str());
+        ostringstream ss;
+        std::string mon_name = mon.monmap->get_name(i);
+        auto const& info = mon.monmap->get(mon_name);
+        if ((rcnow - info.time_added) > mon_down_added_grace) {
+          mon_downs.insert(mon_name);
+         ss << "mon." << mon_name << " (rank " << i
+            << ") addr " << mon.monmap->get_addrs(i)
+            << " is down (out of quorum)";
+         details.push_back(ss.str());
+        }
       }
     }
+    if (details.size()) {
+      ostringstream ss;
+      ss << (max-actual) << "/" << max << " mons down, quorum " << mon.get_quorum_names();
+      auto& d = checks->add("MON_DOWN", HEALTH_WARN, ss.str(), max - actual);
+      d.detail = std::move(details);
+    }
   }
 }