From 181b215abe3c2db8d15365b3afe5da87fe209689 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Wed, 2 May 2018 10:23:33 +0800 Subject: [PATCH] mds: properly reconnect client caps after loading inodes Commit e43c02d6 "mds: filter out blacklisted clients when importing caps" makes MDCache::process_imported_caps() ignore clients that are not in MDCache::rejoin_imported_session_map. The map does not contain clients from which mds has received reconnect messages. This causes some client caps (corresponding inodes were not in cache when mds was in reconnect state) to get dropped. Signed-off-by: "Yan, Zheng" (cherry picked from commit 48f60e7f274de9d76499816a528eff859bb161e3) Conflicts: src/mds/MDCache.h: Resolved for rejoin_recovered_client and rejoin_open_sessions_finish --- src/mds/MDCache.cc | 34 +++++++++++++++++----------------- src/mds/MDCache.h | 9 ++++++--- src/mds/Server.cc | 1 + 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 13056db3065c..4dca0c4cc46a 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -3167,7 +3167,7 @@ void MDCache::handle_resolve(MMDSResolve *m) Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v)); if (session) - rejoin_imported_client_map.emplace(q->first, session->info.inst); + rejoin_client_map.emplace(q->first, session->info.inst); } // will process these caps in rejoin stage @@ -4312,7 +4312,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak) assert(gather_locks.empty()); // check cap exports. - rejoin_imported_client_map.insert(weak->client_map.begin(), weak->client_map.end()); + rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end()); for (auto p = weak->cap_exports.begin(); p != weak->cap_exports.end(); ++p) { CInode *in = get_inode(p->first); @@ -5245,19 +5245,19 @@ void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret) class C_MDC_RejoinSessionsOpened : public MDCacheLogContext { public: - map > imported_session_map; + map > session_map; C_MDC_RejoinSessionsOpened(MDCache *c) : MDCacheLogContext(c) {} void finish(int r) override { assert(r == 0); - mdcache->rejoin_open_sessions_finish(imported_session_map); + mdcache->rejoin_open_sessions_finish(session_map); } }; -void MDCache::rejoin_open_sessions_finish(map >& imported_session_map) +void MDCache::rejoin_open_sessions_finish(map >& session_map) { dout(10) << "rejoin_open_sessions_finish" << dendl; - mds->server->finish_force_open_sessions(imported_session_map); - rejoin_imported_session_map.swap(imported_session_map); + mds->server->finish_force_open_sessions(session_map); + rejoin_session_map.swap(session_map); if (rejoin_gather.empty()) rejoin_gather_finish(); } @@ -5288,14 +5288,14 @@ bool MDCache::process_imported_caps() // called by rejoin_gather_finish() ? if (rejoin_gather.count(mds->get_nodeid()) == 0) { - if (!rejoin_imported_client_map.empty() && - rejoin_imported_session_map.empty()) { + if (!rejoin_client_map.empty() && + rejoin_session_map.empty()) { C_MDC_RejoinSessionsOpened *finish = new C_MDC_RejoinSessionsOpened(this); - version_t pv = mds->server->prepare_force_open_sessions(rejoin_imported_client_map, - finish->imported_session_map); - mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_imported_client_map), finish); + version_t pv = mds->server->prepare_force_open_sessions(rejoin_client_map, + finish->session_map); + mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_client_map), finish); mds->mdlog->flush(); - rejoin_imported_client_map.clear(); + rejoin_client_map.clear(); return true; } @@ -5308,8 +5308,8 @@ bool MDCache::process_imported_caps() for (map::iterator q = p->second.second.begin(); q != p->second.second.end(); ++q) { - auto r = rejoin_imported_session_map.find(q->first); - if (r == rejoin_imported_session_map.end()) + auto r = rejoin_session_map.find(q->first); + if (r == rejoin_session_map.end()) continue; Session *session = r->second.first; @@ -5344,8 +5344,8 @@ bool MDCache::process_imported_caps() for (auto q = p->second.begin(); q != p->second.end(); ++q) { Session *session; { - auto r = rejoin_imported_session_map.find(q->first); - session = (r != rejoin_imported_session_map.end() ? r->second.first : nullptr); + auto r = rejoin_session_map.find(q->first); + session = (r != rejoin_session_map.end() ? r->second.first : nullptr); } for (auto r = q->second.begin(); r != q->second.end(); ++r) { diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index d50bef5f0630..4597169c596a 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -546,8 +546,8 @@ protected: set rejoin_ack_gather; // nodes from whom i need a rejoin ack map > > rejoin_imported_caps; map > > rejoin_slave_exports; - map rejoin_imported_client_map; - map > rejoin_imported_session_map; + map rejoin_client_map; + map > rejoin_session_map; map > > cap_exports; // ino -> target, client -> capex @@ -597,6 +597,9 @@ public: mds_rank_t frommds=MDS_RANK_NONE) { cap_imports[ino][client][frommds] = icr; } + void rejoin_recovered_client(client_t client, const entity_inst_t& inst) { + rejoin_client_map.emplace(client, inst); + } const cap_reconnect_t *get_replay_cap_reconnect(inodeno_t ino, client_t client) { if (cap_imports.count(ino) && cap_imports[ino].count(client) && @@ -641,7 +644,7 @@ public: friend class C_MDC_RejoinOpenInoFinish; friend class C_MDC_RejoinSessionsOpened; void rejoin_open_ino_finish(inodeno_t ino, int ret); - void rejoin_open_sessions_finish(map >& imported_session_map); + void rejoin_open_sessions_finish(map >& session_map); bool process_imported_caps(); void choose_lock_states_and_reconnect_caps(); void prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino, diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 2a402dcfdb9e..a85aeebfc958 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -1004,6 +1004,7 @@ void Server::handle_client_reconnect(MClientReconnect *m) mdcache->rejoin_recovered_caps(p->first, from, p->second, MDS_RANK_NONE); } } + mdcache->rejoin_recovered_client(session->get_client(), session->info.inst); // remove from gather set client_reconnect_gather.erase(from); -- 2.47.3