From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 8 Mar 2023 19:54:04 +0000 (-0500)
Subject: mgr: force propose whenever the active changes
X-Git-Tag: v18.1.0~132^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fb6f85d3dc4af9af7f2fce5498e236ff69e42045;p=ceph.git

mgr: force propose whenever the active changes

This race fixed by 23c3f7 exists wherever we drop the active mgr.
Resolve this by forcing immediate proposal (circumventing any delays)
whenever the active is dropped.

Fixes: 23c3f76018b446fb77bbd71fdd33bddfbae9e06d
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 30b20d3c13be532e5508dc65276e90c7973c0c51)
---

diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
index c06a691b04ad..6f73d8468d25 100644
--- a/src/mon/MgrMap.h
+++ b/src/mon/MgrMap.h
@@ -257,6 +257,13 @@ public:
   // running modules on the active mgr daemon.
   std::map<std::string, std::string> services;
 
+  static MgrMap create_null_mgrmap() {
+    MgrMap null_map;
+    /* Use the largest epoch so it's always bigger than whatever the mgr has. */
+    null_map.epoch = std::numeric_limits<decltype(epoch)>::max();
+    return null_map;
+  }
+
   epoch_t get_epoch() const { return epoch; }
   epoch_t get_last_failure_osd_epoch() const { return last_failure_osd_epoch; }
   const entity_addrvec_t& get_active_addrs() const { return active_addrs; }
diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
index 271c86cdb9a0..2882dffcf872 100644
--- a/src/mon/MgrMonitor.cc
+++ b/src/mon/MgrMonitor.cc
@@ -588,6 +588,11 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
       pending_map.clients = clients;
       updated = true;
     }
+  } else if (m->get_available()) {
+    dout(4) << "mgr thinks it is active but it is not, dropping!" << dendl;
+    auto m = make_message<MMgrMap>(MgrMap::create_null_mgrmap());
+    mon.send_reply(op, m.detach());
+    return true;
   } else if (pending_map.active_gid == 0) {
     // There is no currently active daemon, select this one.
     if (pending_map.standbys.count(m->get_gid())) {
@@ -926,6 +931,12 @@ void MgrMonitor::drop_active()
   pending_map.clients.clear();
   pending_map.last_failure_osd_epoch = blocklist_epoch;
 
+  /* If we are dropping the active, we need to notify clients immediately.
+   * Additionally, avoid logical races with ::prepare_beacon which cannot
+   * accurately determine if a mgr is a standby or an old active.
+   */
+  force_immediate_propose();
+
   // So that when new active mgr subscribes to mgrdigest, it will
   // get an immediate response instead of waiting for next timer
   cancel_timer();
@@ -1286,22 +1297,15 @@ out:
   getline(ss, rs);
 
   if (r >= 0) {
-    bool do_update = false;
-    if (prefix == "mgr fail" && is_writeable()) {
-      propose_pending();
-      do_update = false;
-    } else {
-      do_update = true;
-    }
     // success.. delay reply
     wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
 					      get_last_committed() + 1));
-    return do_update;
   } else {
     // reply immediately
     mon.reply_command(op, r, rs, rdata, get_last_committed());
-    return false;
   }
+
+  return r >= 0;
 }
 
 void MgrMonitor::init()
diff --git a/src/mon/PaxosService.cc b/src/mon/PaxosService.cc
index 0a6a9a9ea70f..cfe11cb5049e 100644
--- a/src/mon/PaxosService.cc
+++ b/src/mon/PaxosService.cc
@@ -100,7 +100,6 @@ bool PaxosService::dispatch(MonOpRequestRef op)
 
   if (need_immediate_propose) {
     dout(10) << __func__ << " forced immediate propose" << dendl;
-    need_immediate_propose = false;
     propose_pending();
     return true;
   }
@@ -224,6 +223,7 @@ void PaxosService::propose_pending()
 
   // apply to paxos
   proposing = true;
+  need_immediate_propose = false; /* reset whenever we propose */
   /**
    * Callback class used to mark us as active once a proposal finishes going
    * through Paxos.