From 286a8863ac82ef1d0a60011cf15192ff34397cf3 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Wed, 25 Feb 2015 15:27:59 +0800 Subject: [PATCH] client: re-send requsets before composing the cap reconnect message After commit 419800fe (client: re-send request when MDS enters reconnecting stage), cephfs client can send both unsafe requests and normal requests when MDS is in reconnecting stage. Normal requests can have embedded cap releases, the client code encodes these embedded cap releases after composing the cap reconnect message. This causes the client sliently drop some caps. The fix is re-send requsets (which add embedded cap releases) before composing the cap reconnect message Fixes: #10912 Signed-off-by: Yan, Zheng (cherry picked from commit 8ea5a811b3b32b99b65e6170976af3d42e6c9ba0) --- src/client/Client.cc | 26 ++++++++++++++------------ src/client/Client.h | 3 ++- src/mds/Server.cc | 3 +++ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index ef11bdb9b4b52..d7b7007b349d2 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -1861,7 +1861,8 @@ void Client::_kick_stale_sessions() } } -void Client::send_request(MetaRequest *request, MetaSession *session) +void Client::send_request(MetaRequest *request, MetaSession *session, + bool drop_cap_releases) { // make the request mds_rank_t mds = session->mds_num; @@ -1875,7 +1876,10 @@ void Client::send_request(MetaRequest *request, MetaSession *session) r->set_replayed_op(); } else { encode_cap_releases(request, mds); - r->releases.swap(request->cap_releases); + if (drop_cap_releases) // we haven't send cap reconnect yet, drop cap releases + request->cap_releases.clear(); + else + r->releases.swap(request->cap_releases); } r->set_mdsmap_epoch(mdsmap->get_epoch()); @@ -2286,6 +2290,13 @@ void Client::send_reconnect(MetaSession *session) session->release = NULL; } + // reset my cap seq number + session->seq = 0; + //connect to the mds' offload targets + connect_mds_targets(mds); + //make sure unsafe requests get saved + resend_unsafe_requests(session); + MClientReconnect *m = new MClientReconnect; // i have an open session. @@ -2324,15 +2335,6 @@ void Client::send_reconnect(MetaSession *session) } } } - - // reset my cap seq number - session->seq = 0; - - //connect to the mds' offload targets - connect_mds_targets(mds); - //make sure unsafe requests get saved - resend_unsafe_requests(session); - session->con->send_message(m); mount_cond.Signal(); @@ -2373,7 +2375,7 @@ void Client::resend_unsafe_requests(MetaSession *session) if (req->retry_attempt == 0) continue; // old requests only if (req->mds == session->mds_num) - send_request(req, session); + send_request(req, session, true); } } diff --git a/src/client/Client.h b/src/client/Client.h index 0d4bb2cf9f5fe..ebca5b2d266c1 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -312,7 +312,8 @@ public: mds_rank_t mds, int drop, int unless); mds_rank_t choose_target_mds(MetaRequest *req); void connect_mds_targets(mds_rank_t mds); - void send_request(MetaRequest *request, MetaSession *session); + void send_request(MetaRequest *request, MetaSession *session, + bool drop_cap_releases=false); MClientRequest *build_client_request(MetaRequest *request); void kick_requests(MetaSession *session); void kick_requests_closed(MetaSession *session); diff --git a/src/mds/Server.cc b/src/mds/Server.cc index edc94cb30066f..d4ed0723962f6 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -130,6 +130,9 @@ void Server::dispatch(Message *m) dout(3) << "queuing completed op" << dendl; queue_replay = true; } + // this request was created before the cap reconnect message, drop any embedded + // cap releases. + req->releases.clear(); } if (queue_replay) { mds->enqueue_replay(new C_MDS_RetryMessage(mds, m)); -- 2.39.5