client: avoid choosing stopped mds as request target

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index ee49792d60b7ea1c58dc9cb3490dc3d5725ec686..8e047f55259c7d22930cf7d7f55a0738601c1e8f 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -811,6 +811,15 @@ void Client::_fragmap_remove_non_leaves(Inode *in)
       ++p;
 }
 
+void Client::_fragmap_remove_stopped_mds(Inode *in, mds_rank_t mds)
+{
+  for (auto p = in->fragmap.begin(); p != in->fragmap.end(); )
+    if (p->second == mds)
+      in->fragmap.erase(p++);
+    else
+      ++p;
+}
+
 Inode * Client::add_update_inode(InodeStat *st, utime_t from,
                                 MetaSession *session,
                                 const UserPerm& request_perms)
@@ -1386,7 +1395,7 @@ Inode* Client::insert_trace(MetaRequest *request, MetaSession *session)
 
 // -------
 
-mds_rank_t Client::choose_target_mds(MetaRequest *req) 
+mds_rank_t Client::choose_target_mds(MetaRequest *req, Inode** phash_diri)
 {
   mds_rank_t mds = MDS_RANK_NONE;
   __u32 hash = 0;
@@ -1455,9 +1464,11 @@ mds_rank_t Client::choose_target_mds(MetaRequest *req)
     if (is_hash && S_ISDIR(in->mode) && !in->fragmap.empty()) {
       frag_t fg = in->dirfragtree[hash];
       if (in->fragmap.count(fg)) {
-        mds = in->fragmap[fg];
-        ldout(cct, 10) << "choose_target_mds from dirfragtree hash" << dendl;
-        goto out;
+       mds = in->fragmap[fg];
+       if (phash_diri)
+         *phash_diri = in;
+       ldout(cct, 10) << "choose_target_mds from dirfragtree hash" << dendl;
+       goto out;
       }
     }
   
@@ -1660,27 +1671,28 @@ int Client::make_request(MetaRequest *request,
     request->caller_cond = &caller_cond;
 
     // choose mds
-    mds_rank_t mds = choose_target_mds(request);
-    if (mds == MDS_RANK_NONE || !mdsmap->is_active_or_stopping(mds)) {
-      ldout(cct, 10) << " target mds." << mds << " not active, waiting for new mdsmap" << dendl;
-      wait_on_list(waiting_for_mdsmap);
+    Inode *hash_diri = NULL;
+    mds_rank_t mds = choose_target_mds(request, &hash_diri);
+    int mds_state = (mds == MDS_RANK_NONE) ? MDSMap::STATE_NULL : mdsmap->get_state(mds);
+    if (mds_state != MDSMap::STATE_ACTIVE && mds_state != MDSMap::STATE_STOPPING) {
+      if (mds_state == MDSMap::STATE_NULL && mds >= mdsmap->get_max_mds()) {
+       if (hash_diri) {
+         ldout(cct, 10) << " target mds." << mds << " has stopped, remove it from fragmap" << dendl;
+         _fragmap_remove_stopped_mds(hash_diri, mds);
+       } else {
+         ldout(cct, 10) << " target mds." << mds << " has stopped, trying a random mds" << dendl;
+         request->resend_mds = _get_random_up_mds();
+       }
+      } else {
+       ldout(cct, 10) << " target mds." << mds << " not active, waiting for new mdsmap" << dendl;
+       wait_on_list(waiting_for_mdsmap);
+      }
       continue;
     }
 
     // open a session?
     MetaSession *session = NULL;
     if (!have_open_session(mds)) {
-      if (!mdsmap->is_active_or_stopping(mds)) {
-       ldout(cct, 10) << "no address for mds." << mds << ", waiting for new mdsmap" << dendl;
-       wait_on_list(waiting_for_mdsmap);
-
-       if (!mdsmap->is_active_or_stopping(mds)) {
-         ldout(cct, 10) << "hmm, still have no address for mds." << mds << ", trying a random mds" << dendl;
-         request->resend_mds = _get_random_up_mds();
-         continue;
-       }
-      }
-      
       session = _get_or_open_mds_session(mds);
 
       // wait

diff --git a/src/client/Client.h b/src/client/Client.h
index 38f51c4f3cc2ffb9e94f9894b5918c55773847c7..412c00d419d6b89282ea252d450b7054c5d17b2a 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -371,7 +371,7 @@ protected:
                           int unless,int force=0);
   void encode_dentry_release(Dentry *dn, MetaRequest *req,
                             mds_rank_t mds, int drop, int unless);
-  mds_rank_t choose_target_mds(MetaRequest *req);
+  mds_rank_t choose_target_mds(MetaRequest *req, Inode** phash_diri=NULL);
   void connect_mds_targets(mds_rank_t mds);
   void send_request(MetaRequest *request, MetaSession *session,
                    bool drop_cap_releases=false);
@@ -736,6 +736,7 @@ private:
 
   // other helpers
   void _fragmap_remove_non_leaves(Inode *in);
+  void _fragmap_remove_stopped_mds(Inode *in, mds_rank_t mds);
 
   void _ll_get(Inode *in);
   int _ll_put(Inode *in, int num);