mon: do not quickly mark mds laggy when MON_DOWN

The MDS may be sending beacons to a partitioned or newly restarted
monitor. This will wrongly cause the current leader to believe the MDS
is unavailable.

Fixes: https://tracker.ceph.com/issues/43216
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in
index 526fc44607e8ee08b212045202f52b78f7da514b..8754f4fb6b80d13117c13900a83a0db23e712ac4 100644 (file)
--- a/src/common/options/mon.yaml.in
+++ b/src/common/options/mon.yaml.in
@@ -764,6 +764,13 @@ options:
   services:
   - mon
   with_legacy: true
+- name: mds_beacon_mon_down_grace
+  type: secs
+  level: advanced
+  desc: tolerance in seconds for missed MDS beacons to monitors
+  fmt_desc: The interval without beacons before Ceph declares an MDS laggy
+    when a monitor is down.
+  default: 1_min
 # skip safety assertions on FSMap (in case of bugs where we want to continue anyway)
 - name: mon_mds_skip_sanity
   type: bool

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 928fce753d0afe1b10ed6ff6884f88d2611dd905..0ad30683e239321ec3473eb07f24708fdedc4847 100644 (file)
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -2136,6 +2136,11 @@ bool MDSMonitor::check_health(FSMap& fsmap, bool* propose_osdmap)
 
   // check beacon timestamps
   std::vector<mds_gid_t> to_remove;
+  const bool mon_down = mon.is_mon_down();
+  const auto mds_beacon_mon_down_grace =
+      g_conf().get_val<std::chrono::seconds>("mds_beacon_mon_down_grace");
+  const auto quorum_age = std::chrono::seconds(mon.quorum_age());
+  const bool new_quorum = quorum_age < mds_beacon_mon_down_grace;
   for (auto it = last_beacon.begin(); it != last_beacon.end(); ) {
     auto& [gid, beacon_info] = *it;
     auto since_last = std::chrono::duration<double>(now-beacon_info.stamp);
@@ -2152,6 +2157,14 @@ bool MDSMonitor::check_health(FSMap& fsmap, bool* propose_osdmap)
               << " (gid: " << gid << " addr: " << info.addrs
               << " state: " << ceph_mds_state_name(info.state) << ")"
               << " since " << since_last.count() << dendl;
+      if ((mon_down || new_quorum) && since_last < mds_beacon_mon_down_grace) {
+        /* The MDS may be sending beacons to a monitor not yet in quorum or
+         * temporarily partitioned. Hold off on removal for a little longer...
+         */
+        dout(10) << "deferring removal for mds_beacon_mon_down_grace during MON_DOWN" << dendl;
+        ++it;
+        continue;
+      }
       // If the OSDMap is writeable, we can blocklist things, so we can
       // try failing any laggy MDS daemons.  Consider each one for failure.
       if (!info.laggy()) {

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 330ecbc70d9507caa5a1623f9f1941d11f61337e..4c9566b396a31ff6a6b175e0b21ca7dce8c83a1d 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -221,6 +221,13 @@ public:
     return age.count();
   }
 
+  bool is_mon_down() const {
+    int max = monmap->size();
+    int actual = get_quorum().size();
+    auto now = ceph::real_clock::now();
+    return actual < max && now > monmap->created.to_real_time();
+  }
+
   // -- elector --
 private:
   std::unique_ptr<Paxos> paxos;