mon: drop forwarded requests after an election

On each election, we resend routed requests to the new leader (or
requeue for ourselves).  Therefore, if we receive a forwarded request,
we should drop it on the floor if there is a new election.  Add a field
in the PaxosServiceMessage struct to track which election epoch we
received the request in, and drop it in PaxosService::dispatch() if
that is in the past.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/messages/PaxosServiceMessage.h b/src/messages/PaxosServiceMessage.h
index fc82e86395cd2c327d49c6d025ff03fb9cb896d2..5132e1c99051043678a679d2c137b0f9adaf3b6d 100644 (file)
--- a/src/messages/PaxosServiceMessage.h
+++ b/src/messages/PaxosServiceMessage.h
@@ -9,13 +9,19 @@ class PaxosServiceMessage : public Message {
   version_t version;
   __s16 session_mon;
   uint64_t session_mon_tid;
+
+  // track which epoch the leader received a forwarded request in, so we can
+  // discard forwarded requests appropriately on election boundaries.
+  epoch_t rx_election_epoch;
   
   PaxosServiceMessage()
     : Message(MSG_PAXOS),
-      version(0), session_mon(-1), session_mon_tid(0) { }
+      version(0), session_mon(-1), session_mon_tid(0),
+      rx_election_epoch(0) { }
   PaxosServiceMessage(int type, version_t v, int enc_version=1, int compat_enc_version=0)
     : Message(type, enc_version, compat_enc_version),
-      version(v), session_mon(-1), session_mon_tid(0) { }
+      version(v), session_mon(-1), session_mon_tid(0),
+      rx_election_epoch(0)  { }
  protected:
   ~PaxosServiceMessage() {}
 

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index e09582d621e52cbf81b368cb9183f5a8c125765e..f2b05419d58106cd3374c70cda1b2601be3b4a21 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2821,6 +2821,14 @@ void Monitor::handle_forward(MForward *m)
     PaxosServiceMessage *req = m->msg;
     m->msg = NULL;  // so ~MForward doesn't delete it
     req->set_connection(c);
+
+    /*
+     * note which election epoch this is; we will drop the message if
+     * there is a future election since our peers will resend routed
+     * requests in that case.
+     */
+    req->rx_election_epoch = get_epoch();
+
     /* Because this is a special fake connection, we need to break
        the ref loop between Connection and MonSession differently
        than we normally do. Here, the Message refers to the Connection

diff --git a/src/mon/PaxosService.cc b/src/mon/PaxosService.cc
index dc5c7d6afeea903da39b39d72ca17b1e655745eb..9484f367a6f119008c7e57b9766910189fc99d93 100644 (file)
--- a/src/mon/PaxosService.cc
+++ b/src/mon/PaxosService.cc
@@ -34,6 +34,16 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon, Paxos *paxos, string
 bool PaxosService::dispatch(PaxosServiceMessage *m)
 {
   dout(10) << "dispatch " << *m << " from " << m->get_orig_source_inst() << dendl;
+
+  // make sure this message isn't forwarded from a previous election epoch
+  if (m->rx_election_epoch &&
+      m->rx_election_epoch < mon->get_epoch()) {
+    dout(10) << " discarding forwarded message from previous election epoch "
+            << m->rx_election_epoch << " < " << mon->get_epoch() << dendl;
+    m->put();
+    return true;
+  }
+
   // make sure our map is readable and up to date
   if (!is_readable(m->version)) {
     dout(10) << " waiting for paxos -> readable (v" << m->version << ")" << dendl;