mds: properly reconnect client caps after loading inodes

Commit e43c02d6 "mds: filter out blacklisted clients when importing
caps" makes MDCache::process_imported_caps() ignore clients that are
not in MDCache::rejoin_imported_session_map. The map does not contain
clients from which mds has received reconnect messages. This causes
some client caps (corresponding inodes were not in cache when mds was
in reconnect state) to get dropped.

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index eb07de00ed1aa27f044df43045401345c163f148..fcd8b5ac573ca7b3a037cd44d52585765f03a898 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -3239,7 +3239,7 @@ void MDCache::handle_resolve(MMDSResolve *m)
 
            Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
            if (session)
-             rejoin_imported_client_map.emplace(q->first, session->info.inst);
+             rejoin_client_map.emplace(q->first, session->info.inst);
          }
 
          // will process these caps in rejoin stage
@@ -4420,7 +4420,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
     assert(gather_locks.empty());
 
     // check cap exports.
-    rejoin_imported_client_map.insert(weak->client_map.begin(), weak->client_map.end());
+    rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end());
 
     for (auto p = weak->cap_exports.begin(); p != weak->cap_exports.end(); ++p) {
       CInode *in = get_inode(p->first);
@@ -5376,19 +5376,19 @@ void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret)
 
 class C_MDC_RejoinSessionsOpened : public MDCacheLogContext {
 public:
-  map<client_t,pair<Session*,uint64_t> > imported_session_map;
+  map<client_t,pair<Session*,uint64_t> > session_map;
   C_MDC_RejoinSessionsOpened(MDCache *c) : MDCacheLogContext(c) {}
   void finish(int r) override {
     assert(r == 0);
-    mdcache->rejoin_open_sessions_finish(imported_session_map);
+    mdcache->rejoin_open_sessions_finish(session_map);
   }
 };
 
-void MDCache::rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& imported_session_map)
+void MDCache::rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& session_map)
 {
   dout(10) << "rejoin_open_sessions_finish" << dendl;
-  mds->server->finish_force_open_sessions(imported_session_map);
-  rejoin_imported_session_map.swap(imported_session_map);
+  mds->server->finish_force_open_sessions(session_map);
+  rejoin_session_map.swap(session_map);
   if (rejoin_gather.empty())
     rejoin_gather_finish();
 }
@@ -5450,14 +5450,14 @@ bool MDCache::process_imported_caps()
 
   // called by rejoin_gather_finish() ?
   if (rejoin_gather.count(mds->get_nodeid()) == 0) {
-    if (!rejoin_imported_client_map.empty() &&
-       rejoin_imported_session_map.empty()) {
+    if (!rejoin_client_map.empty() &&
+       rejoin_session_map.empty()) {
       C_MDC_RejoinSessionsOpened *finish = new C_MDC_RejoinSessionsOpened(this);
-      version_t pv = mds->server->prepare_force_open_sessions(rejoin_imported_client_map,
-                                                             finish->imported_session_map);
-      mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_imported_client_map), finish);
+      version_t pv = mds->server->prepare_force_open_sessions(rejoin_client_map,
+                                                             finish->session_map);
+      mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_client_map), finish);
       mds->mdlog->flush();
-      rejoin_imported_client_map.clear();
+      rejoin_client_map.clear();
       return true;
     }
 
@@ -5470,8 +5470,8 @@ bool MDCache::process_imported_caps()
       for (map<client_t,Capability::Export>::iterator q = p->second.second.begin();
           q != p->second.second.end();
           ++q) {
-       auto r = rejoin_imported_session_map.find(q->first);
-       if (r == rejoin_imported_session_map.end())
+       auto r = rejoin_session_map.find(q->first);
+       if (r == rejoin_session_map.end())
          continue;
 
        Session *session = r->second.first;
@@ -5509,8 +5509,8 @@ bool MDCache::process_imported_caps()
       for (auto q = p->second.begin(); q != p->second.end(); ++q) {
        Session *session;
        {
-         auto r = rejoin_imported_session_map.find(q->first);
-         session = (r != rejoin_imported_session_map.end() ? r->second.first : nullptr);
+         auto r = rejoin_session_map.find(q->first);
+         session = (r != rejoin_session_map.end() ? r->second.first : nullptr);
        }
 
        for (auto r = q->second.begin(); r != q->second.end(); ++r) {

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 99f0b13a8352248ab52f6d782a1bb4aaf251e9ab..3113ac8303c9239a2cd1729e282a22d241a9a6ec 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -552,8 +552,8 @@ protected:
   set<mds_rank_t> rejoin_ack_gather;  // nodes from whom i need a rejoin ack
   map<mds_rank_t,map<inodeno_t,map<client_t,Capability::Import> > > rejoin_imported_caps;
   map<inodeno_t,pair<mds_rank_t,map<client_t,Capability::Export> > > rejoin_slave_exports;
-  map<client_t,entity_inst_t> rejoin_imported_client_map;
-  map<client_t,pair<Session*,uint64_t> > rejoin_imported_session_map;
+  map<client_t,entity_inst_t> rejoin_client_map;
+  map<client_t,pair<Session*,uint64_t> > rejoin_session_map;
 
   map<inodeno_t,pair<mds_rank_t,map<client_t,cap_reconnect_t> > > cap_exports; // ino -> target, client -> capex
 
@@ -603,6 +603,9 @@ public:
                             mds_rank_t frommds=MDS_RANK_NONE) {
     cap_imports[ino][client][frommds] = icr;
   }
+  void rejoin_recovered_client(client_t client, const entity_inst_t& inst) {
+    rejoin_client_map.emplace(client, inst);
+  }
   bool rejoin_has_cap_reconnect(inodeno_t ino) const {
     return cap_imports.count(ino);
   }
@@ -657,7 +660,7 @@ public:
   friend class C_MDC_RejoinSessionsOpened;
   void rejoin_open_ino_finish(inodeno_t ino, int ret);
   void rejoin_prefetch_ino_finish(inodeno_t ino, int ret);
-  void rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& imported_session_map);
+  void rejoin_open_sessions_finish(map<client_t,pair<Session*,uint64_t> >& session_map);
   bool process_imported_caps();
   void choose_lock_states_and_reconnect_caps();
   void prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino,

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 347a5c33ff4cf42adc7b0a5c8eed224130bf1be4..d93ad9d125bda0ac4fc5e8f915ed268b6b4b53a6 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1013,6 +1013,7 @@ void Server::handle_client_reconnect(MClientReconnect *m)
       mdcache->rejoin_recovered_caps(p->first, from, p->second, MDS_RANK_NONE);
     }
   }
+  mdcache->rejoin_recovered_client(session->get_client(), session->info.inst);
 
   // remove from gather set
   client_reconnect_gather.erase(from);