mon/HealthMonitor: avoid MON_DOWN for freshly added Monitor

In testing, we often have the scenario where cephadm has created a
cluster but doesn't add more monitors until well past
mon_down_mkfs_grace. This causes useless MON_DOWN warnings to be thrown
which fails QA jobs. Avoid this situation entirely by giving a
reasonable grace period for a monitor added to the MonMap to join
quorum.

Fixes: https://tracker.ceph.com/issues/73934
Signed-off-by: Patrick Donnelly <pdonnell@ibm.com>
(cherry picked from commit b028a41e1f000b87aab3f263ab3259a0ca439555)

diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in
index 1057aba15e78b599366fe3d776dac62cdf1e6bf2..150c19ca9bb465ccbd947609db616a7fe6079900 100644 (file)
--- a/src/common/options/mon.yaml.in
+++ b/src/common/options/mon.yaml.in
@@ -57,6 +57,15 @@ options:
   - mon
   flags:
   - runtime
+- name: mon_down_added_grace
+  type: secs
+  level: advanced
+  desc: Period in seconds that the cluster may have a newly added mon down
+  default: 3_min
+  services:
+  - mon
+  flags:
+  - runtime
 - name: mon_down_uptime_grace
   type: secs
   level: advanced

diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index 117e4e050e5043fc63a10e6b726064212f4cf4d2..659d9ad2b3eb8c92300be7aafcfd2c057e7c917a 100644 (file)
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -809,30 +809,38 @@ void HealthMonitor::check_for_mon_down(health_check_map_t *checks, std::set<std:
 {
   int max = mon.monmap->size();
   int actual = mon.get_quorum().size();
+  const auto mon_down_mkfs_grace = g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace");
+  const auto mon_down_uptime_grace = g_conf().get_val<std::chrono::seconds>("mon_down_uptime_grace");
+  const auto mon_down_added_grace = g_conf().get_val<std::chrono::seconds>("mon_down_added_grace");
+
   const auto rcnow = ceph::real_clock::now();
   const auto created = mon.monmap->created.to_real_time();
   const auto mcnow = ceph::coarse_mono_clock::now();
   const auto starttime = mon.get_starttime();
 
-  if (actual < max &&
-      (rcnow - created) > g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace") &&
-      (mcnow - starttime) > g_conf().get_val<std::chrono::seconds>("mon_down_uptime_grace")) {
-    ostringstream ss;
-    ss << (max-actual) << "/" << max << " mons down, quorum "
-       << mon.get_quorum_names();
-    auto& d = checks->add("MON_DOWN", HEALTH_WARN, ss.str(), max - actual);
-    set<int> q = mon.get_quorum();
+  if (actual < max && ((rcnow - created) > mon_down_mkfs_grace) && ((mcnow - starttime) > mon_down_uptime_grace)) {
+    auto q = mon.get_quorum();
+    std::list<std::string> details;
     for (int i=0; i<max; i++) {
       if (q.count(i) == 0) {
-       ostringstream ss;
-  std::string mon_name = mon.monmap->get_name(i);
-  mon_downs.insert(mon_name);
-       ss << "mon." << mon_name << " (rank " << i
-          << ") addr " << mon.monmap->get_addrs(i)
-          << " is down (out of quorum)";
-       d.detail.push_back(ss.str());
+        ostringstream ss;
+        std::string mon_name = mon.monmap->get_name(i);
+        auto const& info = mon.monmap->get(mon_name);
+        if ((rcnow - info.time_added) > mon_down_added_grace) {
+          mon_downs.insert(mon_name);
+         ss << "mon." << mon_name << " (rank " << i
+            << ") addr " << mon.monmap->get_addrs(i)
+            << " is down (out of quorum)";
+         details.push_back(ss.str());
+        }
       }
     }
+    if (details.size()) {
+      ostringstream ss;
+      ss << (max-actual) << "/" << max << " mons down, quorum " << mon.get_quorum_names();
+      auto& d = checks->add("MON_DOWN", HEALTH_WARN, ss.str(), max - actual);
+      d.detail = std::move(details);
+    }
   }
 }