client: re-send requsets before composing the cap reconnect message

After commit 419800fe (client: re-send request when MDS enters reconnecting
stage), cephfs client can send both unsafe requests and normal requests when
MDS is in reconnecting stage. Normal requests can have embedded cap releases,
the client code encodes these embedded cap releases after composing the cap
reconnect message. This causes the client sliently drop some caps. The fix
is re-send requsets (which add embedded cap releases) before composing the
cap reconnect message

Fixes: #10912
Signed-off-by: Yan, Zheng <zyan@redhat.com>
(cherry picked from commit 8ea5a811b3b32b99b65e6170976af3d42e6c9ba0)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index ef11bdb9b4b52ab67516e44ebed626b16ada94cf..d7b7007b349d2de74c3b116ab27de9bea936a2f6 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -1861,7 +1861,8 @@ void Client::_kick_stale_sessions()
   }
 }
 
-void Client::send_request(MetaRequest *request, MetaSession *session)
+void Client::send_request(MetaRequest *request, MetaSession *session,
+                         bool drop_cap_releases)
 {
   // make the request
   mds_rank_t mds = session->mds_num;
@@ -1875,7 +1876,10 @@ void Client::send_request(MetaRequest *request, MetaSession *session)
     r->set_replayed_op();
   } else {
     encode_cap_releases(request, mds);
-    r->releases.swap(request->cap_releases);
+    if (drop_cap_releases) // we haven't send cap reconnect yet, drop cap releases
+      request->cap_releases.clear();
+    else
+      r->releases.swap(request->cap_releases);
   }
   r->set_mdsmap_epoch(mdsmap->get_epoch());
 
@@ -2286,6 +2290,13 @@ void Client::send_reconnect(MetaSession *session)
     session->release = NULL;
   }
 
+  // reset my cap seq number
+  session->seq = 0;
+  //connect to the mds' offload targets
+  connect_mds_targets(mds);
+  //make sure unsafe requests get saved
+  resend_unsafe_requests(session);
+
   MClientReconnect *m = new MClientReconnect;
 
   // i have an open session.
@@ -2324,15 +2335,6 @@ void Client::send_reconnect(MetaSession *session)
       }        
     }
   }
-  
-  // reset my cap seq number
-  session->seq = 0;
-  
-  //connect to the mds' offload targets
-  connect_mds_targets(mds);
-  //make sure unsafe requests get saved
-  resend_unsafe_requests(session);
-
   session->con->send_message(m);
 
   mount_cond.Signal();
@@ -2373,7 +2375,7 @@ void Client::resend_unsafe_requests(MetaSession *session)
     if (req->retry_attempt == 0)
       continue; // old requests only
     if (req->mds == session->mds_num)
-      send_request(req, session);
+      send_request(req, session, true);
   }
 }
 

diff --git a/src/client/Client.h b/src/client/Client.h
index 0d4bb2cf9f5fef4c4069c818b2cb32d35f1dcd7f..ebca5b2d266c19df5e204f2a526685e960f9ac72 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -312,7 +312,8 @@ public:
                             mds_rank_t mds, int drop, int unless);
   mds_rank_t choose_target_mds(MetaRequest *req);
   void connect_mds_targets(mds_rank_t mds);
-  void send_request(MetaRequest *request, MetaSession *session);
+  void send_request(MetaRequest *request, MetaSession *session,
+                   bool drop_cap_releases=false);
   MClientRequest *build_client_request(MetaRequest *request);
   void kick_requests(MetaSession *session);
   void kick_requests_closed(MetaSession *session);

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index edc94cb30066f6a7cf183e2189530e6373d9e2bb..d4ed0723962f6e1cddfe3d02131c6f405ead3292 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -130,6 +130,9 @@ void Server::dispatch(Message *m)
          dout(3) << "queuing completed op" << dendl;
          queue_replay = true;
        }
+       // this request was created before the cap reconnect message, drop any embedded
+       // cap releases.
+       req->releases.clear();
       }
       if (queue_replay) {
        mds->enqueue_replay(new C_MDS_RetryMessage(mds, m));