From: Yan, Zheng <zyan@redhat.com>
Date: Mon, 29 Oct 2018 03:03:00 +0000 (+0800)
Subject: mds: handle state change race
X-Git-Tag: v12.2.12~108^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F26005%2Fhead;p=ceph.git

mds: handle state change race

In multi-mds cluster, recovering mds may receive mdsmap that changes
its state after other mds. Furthermore, the recovering mds may receive
messages tiggered by its state change from other mds before it receive
corresponding mdsmap.

Fixes: http://tracker.ceph.com/issues/37594
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit d3a444473abc98e5ce8121af24538a141a292777)

 Conflicts:
	src/mds/Locker.cc
	src/mds/MDCache.cc
	src/mds/MDSRank.h
---

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 42b47087f24b..f67047c8ab1a 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2257,7 +2257,13 @@ void Locker::request_inode_file_caps(CInode *in)
 void Locker::handle_inode_file_caps(MInodeFileCaps *m)
 {
   // nobody should be talking to us during recovery.
-  assert(mds->is_clientreplay() || mds->is_active() || mds->is_stopping());
+  if (mds->get_state() < MDSMap::STATE_CLIENTREPLAY) {
+    if (mds->get_want_state() >= MDSMap::STATE_CLIENTREPLAY) {
+      mds->wait_for_replay(new C_MDS_RetryMessage(mds, m));
+      return;
+    }
+    assert(!"got unexpected message during recovery");
+  }
 
   // ok
   CInode *in = mdcache->get_inode(m->get_ino());
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index d7b40a3126f6..e07df26063fa 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -4643,7 +4643,13 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
   mds_rank_t from = mds_rank_t(strong->get_source().num());
 
   // only a recovering node will get a strong rejoin.
-  assert(mds->is_rejoin());
+  if (!mds->is_rejoin()) {
+    if (mds->get_want_state() == MDSMap::STATE_REJOIN) {
+      mds->wait_for_rejoin(new C_MDS_RetryMessage(mds, strong));
+      return;
+    }
+    assert(!"got unexpected rejoin message during recovery");
+  }
 
   // assimilate any potentially dirty scatterlock state
   for (map<inodeno_t,MMDSCacheRejoin::lock_bls>::iterator p = strong->inode_scatterlocks.begin();
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 3e3ac1d0968c..e76d1c79a6ec 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1817,6 +1817,7 @@ void MDSRank::rejoin_start()
 {
   dout(1) << "rejoin_start" << dendl;
   mdcache->rejoin_start(new C_MDS_VoidFn(this, &MDSRank::rejoin_done));
+  finish_contexts(g_ceph_context, waiting_for_rejoin);
 }
 void MDSRank::rejoin_done()
 {
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 46f72a4b964b..6be660d10655 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -269,7 +269,8 @@ class MDSRank {
 
     ceph_tid_t last_tid;    // for mds-initiated requests (e.g. stray rename)
 
-    list<MDSInternalContextBase*> waiting_for_active, waiting_for_replay, waiting_for_reconnect, waiting_for_resolve;
+    list<MDSInternalContextBase*> waiting_for_active, waiting_for_replay, waiting_for_rejoin,
+				  waiting_for_reconnect, waiting_for_resolve;
     list<MDSInternalContextBase*> waiting_for_any_client_connection;
     list<MDSInternalContextBase*> replay_queue;
     map<mds_rank_t, list<MDSInternalContextBase*> > waiting_for_active_peer;
@@ -409,6 +410,9 @@ class MDSRank {
     void wait_for_replay(MDSInternalContextBase *c) { 
       waiting_for_replay.push_back(c); 
     }
+    void wait_for_rejoin(MDSInternalContextBase *c) {
+      waiting_for_rejoin.push_back(c);
+    }
     void wait_for_reconnect(MDSInternalContextBase *c) {
       waiting_for_reconnect.push_back(c);
     }