mgr: force propose whenever the active changes

This race fixed by 23c3f7 exists wherever we drop the active mgr.
Resolve this by forcing immediate proposal (circumventing any delays)
whenever the active is dropped.

Fixes: 23c3f76018b446fb77bbd71fdd33bddfbae9e06d
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 30b20d3c13be532e5508dc65276e90c7973c0c51)

diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
index c06a691b04add583673b3f46f57825734ce02486..6f73d8468d2571da59ad45dd39c9cc483a4c62cf 100644 (file)
--- a/src/mon/MgrMap.h
+++ b/src/mon/MgrMap.h
@@ -257,6 +257,13 @@ public:
   // running modules on the active mgr daemon.
   std::map<std::string, std::string> services;
 
+  static MgrMap create_null_mgrmap() {
+    MgrMap null_map;
+    /* Use the largest epoch so it's always bigger than whatever the mgr has. */
+    null_map.epoch = std::numeric_limits<decltype(epoch)>::max();
+    return null_map;
+  }
+
   epoch_t get_epoch() const { return epoch; }
   epoch_t get_last_failure_osd_epoch() const { return last_failure_osd_epoch; }
   const entity_addrvec_t& get_active_addrs() const { return active_addrs; }

diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
index 422724e70809463b7c506536e302f2b8a5ca7c3d..10bdcb4e09679012db903802485d79d3658a2a36 100644 (file)
--- a/src/mon/MgrMonitor.cc
+++ b/src/mon/MgrMonitor.cc
@@ -574,6 +574,11 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
       pending_map.clients = clients;
       updated = true;
     }
+  } else if (m->get_available()) {
+    dout(4) << "mgr thinks it is active but it is not, dropping!" << dendl;
+    auto m = make_message<MMgrMap>(MgrMap::create_null_mgrmap());
+    mon.send_reply(op, m.detach());
+    return true;
   } else if (pending_map.active_gid == 0) {
     // There is no currently active daemon, select this one.
     if (pending_map.standbys.count(m->get_gid())) {
@@ -912,6 +917,12 @@ void MgrMonitor::drop_active()
   pending_map.clients.clear();
   pending_map.last_failure_osd_epoch = blocklist_epoch;
 
+  /* If we are dropping the active, we need to notify clients immediately.
+   * Additionally, avoid logical races with ::prepare_beacon which cannot
+   * accurately determine if a mgr is a standby or an old active.
+   */
+  force_immediate_propose();
+
   // So that when new active mgr subscribes to mgrdigest, it will
   // get an immediate response instead of waiting for next timer
   cancel_timer();
@@ -1272,22 +1283,15 @@ out:
   getline(ss, rs);
 
   if (r >= 0) {
-    bool do_update = false;
-    if (prefix == "mgr fail" && is_writeable()) {
-      propose_pending();
-      do_update = false;
-    } else {
-      do_update = true;
-    }
     // success.. delay reply
     wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
                                              get_last_committed() + 1));
-    return do_update;
   } else {
     // reply immediately
     mon.reply_command(op, r, rs, rdata, get_last_committed());
-    return false;
   }
+
+  return r >= 0;
 }
 
 void MgrMonitor::init()

diff --git a/src/mon/PaxosService.cc b/src/mon/PaxosService.cc
index 0a6a9a9ea70f6afd96e8ed0da4ffa9e3e565d88e..cfe11cb5049e836bd9529938db4bf72ace2e90aa 100644 (file)
--- a/src/mon/PaxosService.cc
+++ b/src/mon/PaxosService.cc
@@ -100,7 +100,6 @@ bool PaxosService::dispatch(MonOpRequestRef op)
 
   if (need_immediate_propose) {
     dout(10) << __func__ << " forced immediate propose" << dendl;
-    need_immediate_propose = false;
     propose_pending();
     return true;
   }
@@ -224,6 +223,7 @@ void PaxosService::propose_pending()
 
   // apply to paxos
   proposing = true;
+  need_immediate_propose = false; /* reset whenever we propose */
   /**
    * Callback class used to mark us as active once a proposal finishes going
    * through Paxos.