From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 27 Aug 2021 01:16:30 +0000 (-0400)
Subject: mon: do not quickly mark mds laggy when MON_DOWN
X-Git-Tag: v16.2.8~88^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e518f0808a924aae4666c77fb054285633c8f214;p=ceph.git

mon: do not quickly mark mds laggy when MON_DOWN

The MDS may be sending beacons to a partitioned or newly restarted
monitor. This will wrongly cause the current leader to believe the MDS
is unavailable.

Fixes: https://tracker.ceph.com/issues/43216
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 89c901a4944158c21eb26d53676a709fa2964c9d)

Conflicts:
- src/common/options/mon.yaml.in

Cherry-pick notes:
- Pacific defines options in common/options.cc vs. yaml files
---

diff --git a/src/common/options.cc b/src/common/options.cc
index 6a2115e15a0a..fad8bdb8d2fb 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -2107,6 +2107,11 @@ std::vector<Option> get_global_options() {
     .add_service("mon")
     .set_description("force mons to trim mdsmaps/fsmaps through this epoch"),
 
+    Option("mds_beacon_mon_down_grace", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
+    .set_default(1_min)
+    .set_description("tolerance in seconds for missed MDS beacons to monitors")
+    .set_long_description("The interval without beacons before Ceph declares an MDS laggy when a monitor is down."),
+
     Option("mon_mds_skip_sanity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(false)
     .add_service("mon")
diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 0e30a14613ea..011b8a81ab3f 100644
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -2125,6 +2125,11 @@ bool MDSMonitor::check_health(FSMap& fsmap, bool* propose_osdmap)
 
   // check beacon timestamps
   std::vector<mds_gid_t> to_remove;
+  const bool mon_down = mon.is_mon_down();
+  const auto mds_beacon_mon_down_grace =
+      g_conf().get_val<std::chrono::seconds>("mds_beacon_mon_down_grace");
+  const auto quorum_age = std::chrono::seconds(mon.quorum_age());
+  const bool new_quorum = quorum_age < mds_beacon_mon_down_grace;
   for (auto it = last_beacon.begin(); it != last_beacon.end(); ) {
     auto& [gid, beacon_info] = *it;
     auto since_last = std::chrono::duration<double>(now-beacon_info.stamp);
@@ -2141,6 +2146,14 @@ bool MDSMonitor::check_health(FSMap& fsmap, bool* propose_osdmap)
               << " (gid: " << gid << " addr: " << info.addrs
               << " state: " << ceph_mds_state_name(info.state) << ")"
               << " since " << since_last.count() << dendl;
+      if ((mon_down || new_quorum) && since_last < mds_beacon_mon_down_grace) {
+        /* The MDS may be sending beacons to a monitor not yet in quorum or
+         * temporarily partitioned. Hold off on removal for a little longer...
+         */
+        dout(10) << "deferring removal for mds_beacon_mon_down_grace during MON_DOWN" << dendl;
+        ++it;
+        continue;
+      }
       // If the OSDMap is writeable, we can blocklist things, so we can
       // try failing any laggy MDS daemons.  Consider each one for failure.
       if (!info.laggy()) {
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index e820e537a617..99762c35b96d 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -221,6 +221,13 @@ public:
     return age.count();
   }
 
+  bool is_mon_down() const {
+    int max = monmap->size();
+    int actual = get_quorum().size();
+    auto now = ceph::real_clock::now();
+    return actual < max && now > monmap->created.to_real_time();
+  }
+
   // -- elector --
 private:
   std::unique_ptr<Paxos> paxos;