mds: complete all the replay op when mds is restarted ,no matter the session is close...

In reconnect phase client sent unsafe requests to mds before reconnect msg. So this pr will handle situation like this:

1. In reconnect phase , client sent unsafe request to mds.
2. It reached reconnect timeout. All sessions without sending reconnect msg in time, some of which may had sent unsafe requests, are marked as closed.
(Another situation is #31668, which will deny all client reconnect msg to speed up reboot).
3.So these unsafe request from session without sending reconnect msg in time or being denied could be handled in clientreplay phase.

fixes:http://tracker.ceph.com/issues/40784

Signed-off-by: Shen Hang <harryshen18@gmail.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index df1d2c9571ce4c1bef83503041c1a7cd62fe2a5a..341b4d5d4ffb22d6681c8797aa4bdd6accda9653 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7867,6 +7867,11 @@ std::vector<Option> get_mds_options() {
     .set_default(true)
     .set_description("additional reply to clients that metadata requests are complete but not yet durable"),
 
+    Option("mds_replay_unsafe_with_closed_session", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(false)
+    .set_flag(Option::FLAG_RUNTIME)
+    .set_description("complete all the replay request when mds is restarted, no matter the session is closed or not"),
+
     Option("mds_default_dir_hash", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(CEPH_STR_HASH_RJENKINS)
     .set_description("hash function to select directory fragment for dentry name"),

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 910b048a007b74ae7df1da776b1adb738f0ec827..aca7f991db5224207ef10d202de1f0446eb8e00c 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1836,7 +1836,7 @@ void Locker::file_update_finish(CInode *in, MutationRef& mut, unsigned flags,
 
   if (ack) {
     Session *session = mds->get_session(client);
-    if (session) {
+    if (session && !session->is_closed()) {
       // "oldest flush tid" > 0 means client uses unique TID for each flush
       if (ack->get_oldest_flush_tid() > 0)
         session->add_completed_flush(ack->get_client_tid());

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 77696da76fc19afff2df38867685da12d918a687..bd80c3c9ce96b062cb6bfefe22079e431c2b8afd 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3672,6 +3672,7 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_recall_warning_decay_rate",
     "mds_request_load_average_decay_rate",
     "mds_session_cache_liveness_decay_rate",
+    "mds_replay_unsafe_with_closed_session",
     NULL
   };
   return KEYS;

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index ea1a0269f0f3dd6c79acaa8a4db4eec5132cb852..33e95721fe81cf0a861ae424987e071f846490b4 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -233,6 +233,7 @@ Server::Server(MDSRank *m) :
   terminating_sessions(false),
   recall_throttle(g_conf().get_val<double>("mds_recall_max_decay_rate"))
 {
+  replay_unsafe_with_closed_session = g_conf().get_val<bool>("mds_replay_unsafe_with_closed_session");
   cap_revoke_eviction_timeout = g_conf().get_val<double>("mds_cap_revoke_eviction_timeout");
   supported_features = feature_bitset_t(CEPHFS_FEATURES_MDS_SUPPORTED);
 }
@@ -245,13 +246,23 @@ void Server::dispatch(const cref_t<Message> &m)
     return;
   }
 
+/*
+ *In reconnect phase, client sent unsafe requests to mds before reconnect msg. Seting sessionclosed_isok will handle scenario like this:
+
+1. In reconnect phase, client sent unsafe requests to mds.
+2. It reached reconnect timeout. All sessions without sending reconnect msg in time, some of which may had sent unsafe requests, are marked as closed.
+(Another situation is #31668, which will deny all client reconnect msg to speed up reboot).
+3.So these unsafe request from session without sending reconnect msg in time or being denied could be handled in clientreplay phase.
+
+*/
+  bool sessionclosed_isok = replay_unsafe_with_closed_session;
   // active?
   // handle_slave_request()/handle_client_session() will wait if necessary
   if (m->get_type() == CEPH_MSG_CLIENT_REQUEST && !mds->is_active()) {
     const auto &req = ref_cast<MClientRequest>(m);
     if (mds->is_reconnect() || mds->get_want_state() == CEPH_MDS_STATE_RECONNECT) {
       Session *session = mds->get_session(req);
-      if (!session || session->is_closed()) {
+      if (!session || (!session->is_open() && !sessionclosed_isok)) {
        dout(5) << "session is closed, dropping " << req->get_reqid() << dendl;
        return;
       }
@@ -819,8 +830,9 @@ void Server::_session_logged(Session *session, uint64_t state_seq, bool open, ve
     } else if (session->is_killing()) {
       // destroy session, close connection
       if (session->get_connection()) {
-       session->get_connection()->mark_down();
-       session->get_connection()->set_priv(NULL);
+        session->get_connection()->mark_down();
+        mds->sessionmap.set_state(session, Session::STATE_CLOSED);
+        session->set_connection(nullptr);
       }
       mds->sessionmap.remove_session(session);
     } else {
@@ -1125,6 +1137,9 @@ void Server::evict_cap_revoke_non_responders() {
 }
 
 void Server::handle_conf_change(const std::set<std::string>& changed) {
+  if (changed.count("mds_replay_unsafe_with_closed_session")) {
+    replay_unsafe_with_closed_session = g_conf().get_val<bool>("mds_replay_unsafe_with_closed_session");
+  }
   if (changed.count("mds_cap_revoke_eviction_timeout")) {
     cap_revoke_eviction_timeout = g_conf().get_val<double>("mds_cap_revoke_eviction_timeout");
     dout(20) << __func__ << " cap revoke eviction timeout changed to "
@@ -1282,6 +1297,14 @@ void Server::handle_client_reconnect(const cref_t<MClientReconnect> &m)
     return;
   }
 
+  if (!session->is_open()) {
+    dout(0) << " ignoring msg from not-open session" << *m << dendl;
+    auto reply = make_message<MClientSession>(CEPH_SESSION_CLOSE);
+    reply->metadata["error_string"] = "session is not open";
+    mds->send_message(reply, m->get_connection());
+    return;
+  }
+
   if (!mds->is_reconnect() && mds->get_want_state() == CEPH_MDS_STATE_RECONNECT) {
     dout(10) << " we're almost in reconnect state (mdsmap delivery race?); waiting" << dendl;
     mds->wait_for_reconnect(new C_MDS_RetryMessage(mds, m));
@@ -2211,13 +2234,14 @@ void Server::handle_client_request(const cref_t<MClientRequest> &req)
     return;
   }
 
+  bool sessionclosed_isok = replay_unsafe_with_closed_session;
   // active session?
   Session *session = 0;
   if (req->get_source().is_client()) {
     session = mds->get_session(req);
     if (!session) {
       dout(5) << "no session for " << req->get_source() << ", dropping" << dendl;
-    } else if (session->is_closed() ||
+    } else if ((session->is_closed() && (!mds->is_clientreplay() || !sessionclosed_isok)) ||
               session->is_closing() ||
               session->is_killing()) {
       dout(5) << "session closed|closing|killing, dropping" << dendl;
@@ -2243,6 +2267,8 @@ void Server::handle_client_request(const cref_t<MClientRequest> &req)
     inodeno_t created;
     if (session->have_completed_request(req->get_reqid().tid, &created)) {
       has_completed = true;
+      if (!session->is_open())
+        return;
       // Don't send traceless reply if the completed request has created
       // new inode. Treat the request as lookup request instead.
       if (req->is_replay() ||
@@ -3209,7 +3235,7 @@ CInode* Server::prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino
   // state. In that corner case, session's prealloc_inos are being freed.
   // To simplify the code, we disallow using/refilling session's prealloc_ino
   // while session is opening.
-  bool allow_prealloc_inos = !mdr->session->is_opening();
+  bool allow_prealloc_inos = mdr->session->is_open();
 
   // assign ino
   if (allow_prealloc_inos &&
@@ -3224,7 +3250,7 @@ CInode* Server::prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino
             << dendl;
   } else {
     mdr->alloc_ino = 
-      in->inode.ino = mds->inotable->project_alloc_id();
+      in->inode.ino = mds->inotable->project_alloc_id(useino);
     dout(10) << "prepare_new_inode alloc " << mdr->alloc_ino << dendl;
   }
 

diff --git a/src/mds/Server.h b/src/mds/Server.h
index f47627c99f7a0bf3025284540d26ddcb9cd7d10c..2e7a34e8a19e750534f6f7cb6282ec487810b44d 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -105,6 +105,7 @@ private:
   feature_bitset_t supported_features;
   feature_bitset_t required_client_features;
 
+  bool replay_unsafe_with_closed_session = false;
   double cap_revoke_eviction_timeout = 0;
 
   friend class MDSContinuation;