mds: properly reconnect client caps after loading inodes

Commit e43c02d6 "mds: filter out blacklisted clients when importing
caps" makes MDCache::process_imported_caps() ignore clients that are
not in MDCache::rejoin_imported_session_map. The map does not contain
clients from which mds has received reconnect messages. This causes
some client caps (corresponding inodes were not in cache when mds was
in reconnect state) to get dropped.

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit 48f60e7f274de9d76499816a528eff859bb161e3)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index e768b315aad294e270f4ca9de51ac9de047c38b1..a7e4d4b6731003e9e0c14ce7e0c378d0af3c948d 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -3242,7 +3242,7 @@ void MDCache::handle_resolve(MMDSResolve *m)
 
            Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
            if (session)
-             rejoin_imported_client_map.emplace(q->first, session->info.inst);
+             rejoin_client_map.emplace(q->first, session->info.inst);
          }
 
          // will process these caps in rejoin stage
@@ -4423,7 +4423,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
     assert(gather_locks.empty());
 
     // check cap exports.
-    rejoin_imported_client_map.insert(weak->client_map.begin(), weak->client_map.end());
+    rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end());
 
     for (auto p = weak->cap_exports.begin(); p != weak->cap_exports.end(); ++p) {
       CInode *in = get_inode(p->first);
@@ -5371,19 +5371,19 @@ void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret)
 
 class C_MDC_RejoinSessionsOpened : public MDCacheLogContext {
 public:
-  map<client_t,pair<Session*,uint64_t> > imported_session_map;
+  map<client_t,pair<Session*,uint64_t> > session_map;
   C_MDC_RejoinSessionsOpened(MDCache *c) : MDCacheLogContext(c) {}
   void finish(int r) override {
     assert(r == 0);
-    mdcache->rejoin_open_sessions_finish(imported_session_map);
+    mdcache->rejoin_open_sessions_finish(session_map);
   }
 };
 
-void MDCache::rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& imported_session_map)
+void MDCache::rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& session_map)
 {
   dout(10) << "rejoin_open_sessions_finish" << dendl;
-  mds->server->finish_force_open_sessions(imported_session_map);
-  rejoin_imported_session_map.swap(imported_session_map);
+  mds->server->finish_force_open_sessions(session_map);
+  rejoin_session_map.swap(session_map);
   if (rejoin_gather.empty())
     rejoin_gather_finish();
 }
@@ -5445,14 +5445,14 @@ bool MDCache::process_imported_caps()
 
   // called by rejoin_gather_finish() ?
   if (rejoin_gather.count(mds->get_nodeid()) == 0) {
-    if (!rejoin_imported_client_map.empty() &&
-       rejoin_imported_session_map.empty()) {
+    if (!rejoin_client_map.empty() &&
+       rejoin_session_map.empty()) {
       C_MDC_RejoinSessionsOpened *finish = new C_MDC_RejoinSessionsOpened(this);
-      version_t pv = mds->server->prepare_force_open_sessions(rejoin_imported_client_map,
-                                                             finish->imported_session_map);
-      mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_imported_client_map), finish);
+      version_t pv = mds->server->prepare_force_open_sessions(rejoin_client_map,
+                                                             finish->session_map);
+      mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_client_map), finish);
       mds->mdlog->flush();
-      rejoin_imported_client_map.clear();
+      rejoin_client_map.clear();
       return true;
     }
 
@@ -5465,8 +5465,8 @@ bool MDCache::process_imported_caps()
       for (map<client_t,Capability::Export>::iterator q = p->second.second.begin();
           q != p->second.second.end();
           ++q) {
-       auto r = rejoin_imported_session_map.find(q->first);
-       if (r == rejoin_imported_session_map.end())
+       auto r = rejoin_session_map.find(q->first);
+       if (r == rejoin_session_map.end())
          continue;
 
        Session *session = r->second.first;
@@ -5504,8 +5504,8 @@ bool MDCache::process_imported_caps()
       for (auto q = p->second.begin(); q != p->second.end(); ++q) {
        Session *session;
        {
-         auto r = rejoin_imported_session_map.find(q->first);
-         session = (r != rejoin_imported_session_map.end() ? r->second.first : nullptr);
+         auto r = rejoin_session_map.find(q->first);
+         session = (r != rejoin_session_map.end() ? r->second.first : nullptr);
        }
 
        for (auto r = q->second.begin(); r != q->second.end(); ++r) {

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 067b7ca2da2da1311cedc1d6769bcff049e74a64..b952c7fe9461b6c595e3e575191a10ead3f7f574 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -552,8 +552,8 @@ protected:
   set<mds_rank_t> rejoin_ack_gather;  // nodes from whom i need a rejoin ack
   map<mds_rank_t,map<inodeno_t,map<client_t,Capability::Import> > > rejoin_imported_caps;
   map<inodeno_t,pair<mds_rank_t,map<client_t,Capability::Export> > > rejoin_slave_exports;
-  map<client_t,entity_inst_t> rejoin_imported_client_map;
-  map<client_t,pair<Session*,uint64_t> > rejoin_imported_session_map;
+  map<client_t,entity_inst_t> rejoin_client_map;
+  map<client_t,pair<Session*,uint64_t> > rejoin_session_map;
 
   map<inodeno_t,pair<mds_rank_t,map<client_t,cap_reconnect_t> > > cap_exports; // ino -> target, client -> capex
 
@@ -603,6 +603,9 @@ public:
                             mds_rank_t frommds=MDS_RANK_NONE) {
     cap_imports[ino][client][frommds] = icr;
   }
+  void rejoin_recovered_client(client_t client, const entity_inst_t& inst) {
+    rejoin_client_map.emplace(client, inst);
+  }
   bool rejoin_has_cap_reconnect(inodeno_t ino) const {
     return cap_imports.count(ino);
   }
@@ -657,7 +660,7 @@ public:
   friend class C_MDC_RejoinSessionsOpened;
   void rejoin_open_ino_finish(inodeno_t ino, int ret);
   void rejoin_prefetch_ino_finish(inodeno_t ino, int ret);
-  void rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& imported_session_map);
+  void rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& session_map);
   bool process_imported_caps();
   void choose_lock_states_and_reconnect_caps();
   void prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino,

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index cba7408d07def957e348087713bc94bdbf7d6147..d68f863c872581782a535a95481060d2ed15c66f 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1013,6 +1013,7 @@ void Server::handle_client_reconnect(MClientReconnect *m)
       mdcache->rejoin_recovered_caps(p->first, from, p->second, MDS_RANK_NONE);
     }
   }
+  mdcache->rejoin_recovered_client(session->get_client(), session->info.inst);
 
   // remove from gather set
   client_reconnect_gather.erase(from);