mds: properly reconnect client caps after loading inodes

Commit e43c02d6 "mds: filter out blacklisted clients when importing
caps" makes MDCache::process_imported_caps() ignore clients that are
not in MDCache::rejoin_imported_session_map. The map does not contain
clients from which mds has received reconnect messages. This causes
some client caps (corresponding inodes were not in cache when mds was
in reconnect state) to get dropped.

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit 48f60e7f274de9d76499816a528eff859bb161e3)

Conflicts:
	src/mds/MDCache.h: Resolved for rejoin_recovered_client and
				rejoin_open_sessions_finish

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 13056db3065cb52d070bdfca8bc4dfaec4e8b1e1..4dca0c4cc46a6c24319a49aa1b8c62a302594bbd 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -3167,7 +3167,7 @@ void MDCache::handle_resolve(MMDSResolve *m)
 
            Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
            if (session)
-             rejoin_imported_client_map.emplace(q->first, session->info.inst);
+             rejoin_client_map.emplace(q->first, session->info.inst);
          }
 
          // will process these caps in rejoin stage
@@ -4312,7 +4312,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
     assert(gather_locks.empty());
 
     // check cap exports.
-    rejoin_imported_client_map.insert(weak->client_map.begin(), weak->client_map.end());
+    rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end());
 
     for (auto p = weak->cap_exports.begin(); p != weak->cap_exports.end(); ++p) {
       CInode *in = get_inode(p->first);
@@ -5245,19 +5245,19 @@ void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret)
 
 class C_MDC_RejoinSessionsOpened : public MDCacheLogContext {
 public:
-  map<client_t,pair<Session*,uint64_t> > imported_session_map;
+  map<client_t,pair<Session*,uint64_t> > session_map;
   C_MDC_RejoinSessionsOpened(MDCache *c) : MDCacheLogContext(c) {}
   void finish(int r) override {
     assert(r == 0);
-    mdcache->rejoin_open_sessions_finish(imported_session_map);
+    mdcache->rejoin_open_sessions_finish(session_map);
   }
 };
 
-void MDCache::rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& imported_session_map)
+void MDCache::rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& session_map)
 {
   dout(10) << "rejoin_open_sessions_finish" << dendl;
-  mds->server->finish_force_open_sessions(imported_session_map);
-  rejoin_imported_session_map.swap(imported_session_map);
+  mds->server->finish_force_open_sessions(session_map);
+  rejoin_session_map.swap(session_map);
   if (rejoin_gather.empty())
     rejoin_gather_finish();
 }
@@ -5288,14 +5288,14 @@ bool MDCache::process_imported_caps()
 
   // called by rejoin_gather_finish() ?
   if (rejoin_gather.count(mds->get_nodeid()) == 0) {
-    if (!rejoin_imported_client_map.empty() &&
-       rejoin_imported_session_map.empty()) {
+    if (!rejoin_client_map.empty() &&
+       rejoin_session_map.empty()) {
       C_MDC_RejoinSessionsOpened *finish = new C_MDC_RejoinSessionsOpened(this);
-      version_t pv = mds->server->prepare_force_open_sessions(rejoin_imported_client_map,
-                                                             finish->imported_session_map);
-      mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_imported_client_map), finish);
+      version_t pv = mds->server->prepare_force_open_sessions(rejoin_client_map,
+                                                             finish->session_map);
+      mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_client_map), finish);
       mds->mdlog->flush();
-      rejoin_imported_client_map.clear();
+      rejoin_client_map.clear();
       return true;
     }
 
@@ -5308,8 +5308,8 @@ bool MDCache::process_imported_caps()
       for (map<client_t,Capability::Export>::iterator q = p->second.second.begin();
           q != p->second.second.end();
           ++q) {
-       auto r = rejoin_imported_session_map.find(q->first);
-       if (r == rejoin_imported_session_map.end())
+       auto r = rejoin_session_map.find(q->first);
+       if (r == rejoin_session_map.end())
          continue;
 
        Session *session = r->second.first;
@@ -5344,8 +5344,8 @@ bool MDCache::process_imported_caps()
       for (auto q = p->second.begin(); q != p->second.end(); ++q) {
        Session *session;
        {
-         auto r = rejoin_imported_session_map.find(q->first);
-         session = (r != rejoin_imported_session_map.end() ? r->second.first : nullptr);
+         auto r = rejoin_session_map.find(q->first);
+         session = (r != rejoin_session_map.end() ? r->second.first : nullptr);
        }
 
        for (auto r = q->second.begin(); r != q->second.end(); ++r) {

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index d50bef5f06305453a5bffeaa73837b594ff128df..4597169c596a8aadba635c1fd137bf8a7d10917b 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -546,8 +546,8 @@ protected:
   set<mds_rank_t> rejoin_ack_gather;  // nodes from whom i need a rejoin ack
   map<mds_rank_t,map<inodeno_t,map<client_t,Capability::Import> > > rejoin_imported_caps;
   map<inodeno_t,pair<mds_rank_t,map<client_t,Capability::Export> > > rejoin_slave_exports;
-  map<client_t,entity_inst_t> rejoin_imported_client_map;
-  map<client_t,pair<Session*,uint64_t> > rejoin_imported_session_map;
+  map<client_t,entity_inst_t> rejoin_client_map;
+  map<client_t,pair<Session*,uint64_t> > rejoin_session_map;
 
   map<inodeno_t,pair<mds_rank_t,map<client_t,cap_reconnect_t> > > cap_exports; // ino -> target, client -> capex
 
@@ -597,6 +597,9 @@ public:
                             mds_rank_t frommds=MDS_RANK_NONE) {
     cap_imports[ino][client][frommds] = icr;
   }
+  void rejoin_recovered_client(client_t client, const entity_inst_t& inst) {
+    rejoin_client_map.emplace(client, inst);
+  }
   const cap_reconnect_t *get_replay_cap_reconnect(inodeno_t ino, client_t client) {
     if (cap_imports.count(ino) &&
        cap_imports[ino].count(client) &&
@@ -641,7 +644,7 @@ public:
   friend class C_MDC_RejoinOpenInoFinish;
   friend class C_MDC_RejoinSessionsOpened;
   void rejoin_open_ino_finish(inodeno_t ino, int ret);
-  void rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& imported_session_map);
+  void rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& session_map);
   bool process_imported_caps();
   void choose_lock_states_and_reconnect_caps();
   void prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino,

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 2a402dcfdb9e3a45c3cb825b4fb2907a8ac00be5..a85aeebfc9580c35ac82a39b473bfbffd9e34557 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1004,6 +1004,7 @@ void Server::handle_client_reconnect(MClientReconnect *m)
       mdcache->rejoin_recovered_caps(p->first, from, p->second, MDS_RANK_NONE);
     }
   }
+  mdcache->rejoin_recovered_client(session->get_client(), session->info.inst);
 
   // remove from gather set
   client_reconnect_gather.erase(from);