client: use session state to track if it's rejected by mds

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 84fcf91d0d926ca5fa9a84c508daf260024cb361..092a799915c1eb67980df1360538e3e6edb546da 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -1721,16 +1721,14 @@ int Client::make_request(MetaRequest *request,
     // open a session?
     if (!have_open_session(mds)) {
       session = _get_or_open_mds_session(mds);
-
+      if (session->state == MetaSession::STATE_REJECTED) {
+       request->abort(-EPERM);
+       break;
+      }
       // wait
       if (session->state == MetaSession::STATE_OPENING) {
        ldout(cct, 10) << "waiting for session to mds." << mds << " to open" << dendl;
        wait_on_context_list(session->waiting_for_open);
-        // Abort requests on REJECT from MDS
-        if (rejected_by_mds.count(mds)) {
-          request->abort(-EPERM);
-          break;
-        }
        continue;
       }
 
@@ -2032,20 +2030,6 @@ MetaSession *Client::_open_mds_session(mds_rank_t mds)
   ceph_assert(em.second); /* not already present */
   MetaSession *session = &em.first->second;
 
-  // Maybe skip sending a request to open if this MDS daemon
-  // has previously sent us a REJECT.
-  if (rejected_by_mds.count(mds)) {
-    if (rejected_by_mds[mds] == session->addrs) {
-      ldout(cct, 4) << __func__ << " mds." << mds << " skipping "
-                       "because we were rejected" << dendl;
-      return session;
-    } else {
-      ldout(cct, 4) << __func__ << " mds." << mds << " old inst "
-                       "rejected us, trying with new inst" << dendl;
-      rejected_by_mds.erase(mds);
-    }
-  }
-
   auto m = make_message<MClientSession>(CEPH_SESSION_REQUEST_OPEN);
   m->metadata = metadata;
   m->supported_features = feature_bitset_t(CEPHFS_FEATURES_CLIENT_SUPPORTED);
@@ -2060,16 +2044,20 @@ void Client::_close_mds_session(MetaSession *s)
   s->con->send_message2(make_message<MClientSession>(CEPH_SESSION_REQUEST_CLOSE, s->seq));
 }
 
-void Client::_closed_mds_session(MetaSession *s)
+void Client::_closed_mds_session(MetaSession *s, bool rejected)
 {
   ldout(cct, 5) << __func__ << " mds." << s->mds_num << " seq " << s->seq << dendl;
-  s->state = MetaSession::STATE_CLOSED;
+  if (rejected && s->state != MetaSession::STATE_CLOSING)
+    s->state = MetaSession::STATE_REJECTED;
+  else
+    s->state = MetaSession::STATE_CLOSED;
   s->con->mark_down();
   signal_context_list(s->waiting_for_open);
   mount_cond.notify_all();
   remove_session_caps(s);
   kick_requests_closed(s);
-  mds_sessions.erase(s->mds_num);
+  if (s->state == MetaSession::STATE_CLOSED)
+    mds_sessions.erase(s->mds_num);
 }
 
 void Client::handle_client_session(const MConstRef<MClientSession>& m)
@@ -2091,9 +2079,8 @@ void Client::handle_client_session(const MConstRef<MClientSession>& m)
       if (!missing_features.empty()) {
        lderr(cct) << "mds." << from << " lacks required features '"
                   << missing_features << "', closing session " << dendl;
-       rejected_by_mds[session->mds_num] = session->addrs;
        _close_mds_session(session);
-       _closed_mds_session(session);
+       _closed_mds_session(session, -EPERM, true);
        break;
       }
       session->mds_features = std::move(m->supported_features);
@@ -2156,8 +2143,7 @@ void Client::handle_client_session(const MConstRef<MClientSession>& m)
        error_str = "unknown error";
       lderr(cct) << "mds." << from << " rejected us (" << error_str << ")" << dendl;
 
-      rejected_by_mds[session->mds_num] = session->addrs;
-      _closed_mds_session(session);
+      _closed_mds_session(session, -EPERM, true);
     }
     break;
 
@@ -2185,6 +2171,10 @@ void Client::_kick_stale_sessions()
 
   for (auto it = mds_sessions.begin(); it != mds_sessions.end(); ) {
     MetaSession &s = it->second;
+    if (s.state == MetaSession::STATE_REJECTED) {
+      mds_sessions.erase(it++);
+      continue;
+    }
     ++it;
     if (s.state == MetaSession::STATE_STALE)
       _closed_mds_session(&s);
@@ -5953,6 +5943,13 @@ int Client::mount(const std::string &mount_root, const UserPerm& perms,
 
 void Client::_close_sessions()
 {
+  for (auto it = mds_sessions.begin(); it != mds_sessions.end(); ) {
+    if (it->second.state == MetaSession::STATE_REJECTED)
+      mds_sessions.erase(it++);
+    else
+      ++it;
+  }
+
   while (!mds_sessions.empty()) {
     // send session closes!
     for (auto &p : mds_sessions) {
@@ -14418,14 +14415,14 @@ int Client::start_reclaim(const std::string& uuid, unsigned flags,
     MetaSession *session;
     if (!have_open_session(mds)) {
       session = _get_or_open_mds_session(mds);
+      if (session->state == MetaSession::STATE_REJECTED)
+       return -EPERM;
       if (session->state != MetaSession::STATE_OPENING) {
        // umounting?
        return -EINVAL;
       }
       ldout(cct, 10) << "waiting for session to mds." << mds << " to open" << dendl;
       wait_on_context_list(session->waiting_for_open);
-      if (rejected_by_mds.count(mds))
-       return -EPERM;
       continue;
     }
 

diff --git a/src/client/Client.h b/src/client/Client.h
index 0ec989db2672b881b9f18c1e4d78804016a41441..83ee00cf844ee08b781153530ce48f7d143f0946 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -783,7 +783,7 @@ protected:
   MetaSession *_get_or_open_mds_session(mds_rank_t mds);
   MetaSession *_open_mds_session(mds_rank_t mds);
   void _close_mds_session(MetaSession *s);
-  void _closed_mds_session(MetaSession *s);
+  void _closed_mds_session(MetaSession *s, bool rejected=false);
   bool _any_stale_sessions() const;
   void _kick_stale_sessions();
   void handle_client_session(const MConstRef<MClientSession>& m);
@@ -1257,12 +1257,6 @@ private:
   ino_t last_used_faked_ino;
   ino_t last_used_faked_root;
 
-  // When an MDS has sent us a REJECT, remember that and don't
-  // contact it again.  Remember which inst rejected us, so that
-  // when we talk to another inst with the same rank we can
-  // try again.
-  std::map<mds_rank_t, entity_addrvec_t> rejected_by_mds;
-
   int local_osd = -ENXIO;
   epoch_t local_osd_epoch = 0;
 

diff --git a/src/client/MetaSession.h b/src/client/MetaSession.h
index 94881c370cac3d78488df5bf69beeeeb04ecd576..c0901305a810481a61181764a3bf7b799b4973a0 100644 (file)
--- a/src/client/MetaSession.h
+++ b/src/client/MetaSession.h
@@ -33,6 +33,7 @@ struct MetaSession {
     STATE_CLOSING,
     STATE_CLOSED,
     STATE_STALE,
+    STATE_REJECTED,
   } state;
 
   enum {