msg: Fix inconsistent message sequence negotiation during connection reset

Backport: firefly, emperor, dumpling

Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
Reviewed-by: Greg Farnum <greg@inktank.com>
(cherry picked from commit bdee119076dd0eb65334840d141ccdf06091e3c9)

diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc
index ae9bb100c82e918ab335ed00f070e6b19d641c02..65db90ac9f3183314249eb02ab8bdcac2bf11ee0 100644 (file)
--- a/src/msg/Pipe.cc
+++ b/src/msg/Pipe.cc
@@ -230,6 +230,10 @@ int Pipe::accept()
   bool authorizer_valid;
   uint64_t feat_missing;
   bool replaced = false;
+  // this variable denotes if the connection attempt from peer is a hard 
+  // reset or not, it is true if there is an existing connection and the
+  // connection sequence from peer is equal to zero
+  bool is_reset_from_peer = false;
   CryptoKey session_key;
   int removed; // single-use down below
 
@@ -444,6 +448,8 @@ int Pipe::accept()
 
       if (connect.connect_seq == 0 && existing->connect_seq > 0) {
        ldout(msgr->cct,0) << "accept peer reset, then tried to connect to us, replacing" << dendl;
+        // this is a hard reset from peer
+        is_reset_from_peer = true;
        if (policy.resetcheck)
          existing->was_session_reset(); // this resets out_queue, msg_ and connect_seq #'s
        goto replace;
@@ -566,7 +572,8 @@ int Pipe::accept()
  replace:
   assert(existing->pipe_lock.is_locked());
   assert(pipe_lock.is_locked());
-  if (connect.features & CEPH_FEATURE_RECONNECT_SEQ) {
+  // if it is a hard reset from peer, we don't need a round-trip to negotiate in/out sequence
+  if ((connect.features & CEPH_FEATURE_RECONNECT_SEQ) && !is_reset_from_peer) {
     reply_tag = CEPH_MSGR_TAG_SEQ;
     existing_seq = existing->in_seq;
   }
@@ -600,7 +607,10 @@ int Pipe::accept()
     uint64_t replaced_conn_id = conn_id;
     conn_id = existing->conn_id;
     existing->conn_id = replaced_conn_id;
-    in_seq = existing->in_seq;
+
+    // reset the in_seq if this is a hard reset from peer,
+    // otherwise we respect our original connection's value
+    in_seq = is_reset_from_peer ? 0 : existing->in_seq;
     in_seq_acked = in_seq;
 
     // steal outgoing queue and out_seq