From: Yan, Zheng Date: Fri, 4 May 2018 02:27:34 +0000 (+0800) Subject: mds: reclaim session before allowing mds to become active X-Git-Tag: v14.0.1~80^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=544159b590f17d3d31376b9ae872621e9c7c7cc7;p=ceph.git mds: reclaim session before allowing mds to become active Recovering MDS does not go to active state when there are sessions that need to be reclaimed. To make mds go to active state, Other client should kill these session by calling: ceph_start_reclaim(cmount, uuid, CEPH_RECLAIM_RESET) Signed-off-by: "Yan, Zheng" --- diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 58ed76055b30..22c779c43175 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -329,6 +329,8 @@ void MDSRankDispatcher::tick() if (is_reconnect()) server->reconnect_tick(); + if (is_clientreplay()) + maybe_clientreplay_done(); if (is_active()) { balancer->tick(); @@ -618,17 +620,6 @@ bool MDSRank::_dispatch(const Message::const_ref &m, bool new_msg) return true; } - // done with all client replayed requests? - if (is_clientreplay() && - mdcache->is_open() && - replay_queue.empty() && - beacon.get_want_state() == MDSMap::STATE_CLIENTREPLAY) { - int num_requests = mdcache->get_num_client_requests(); - dout(10) << " still have " << num_requests << " active replay requests" << dendl; - if (num_requests == 0) - clientreplay_done(); - } - // hack: thrash exports static utime_t start; utime_t now = ceph_clock_now(); @@ -1481,7 +1472,7 @@ void MDSRank::rejoin_done() return; } - if (replay_queue.empty()) + if (replay_queue.empty() && !server->get_num_pending_reclaim()) request_state(MDSMap::STATE_ACTIVE); else request_state(MDSMap::STATE_CLIENTREPLAY); @@ -1497,13 +1488,31 @@ void MDSRank::clientreplay_start() bool MDSRank::queue_one_replay() { - if (replay_queue.empty()) { + if (!replay_queue.empty()) { + queue_waiter(replay_queue.front()); + replay_queue.pop_front(); + return true; + } + // don't go to active if there are session waiting for being reclaimed + if (!server->get_num_pending_reclaim()) mdlog->wait_for_safe(new C_MDS_VoidFn(this, &MDSRank::clientreplay_done)); - return false; + return false; +} + +void MDSRank::maybe_clientreplay_done() +{ + if (is_clientreplay() && + get_want_state() == MDSMap::STATE_CLIENTREPLAY && + replay_queue.empty()) { + int num_requests = mdcache->get_num_client_requests(); + int num_reclaim = server->get_num_pending_reclaim(); + if (!num_requests && !num_reclaim) { + clientreplay_done(); + } else { + dout(1) << " still have " << num_requests << " active replay requests, " + << num_reclaim << " sessions need to be reclaimed" << dendl; + } } - queue_waiter(replay_queue.front()); - replay_queue.pop_front(); - return true; } void MDSRank::clientreplay_done() diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index 69a1fb9269f1..9055f39c2959 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -421,6 +421,7 @@ class MDSRank { } bool queue_one_replay(); + void maybe_clientreplay_done(); void set_osd_epoch_barrier(epoch_t e); epoch_t get_osd_epoch_barrier() const {return osd_epoch_barrier;} diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 956a2adbe654..9dbd5924235f 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -704,15 +704,20 @@ void Server::_session_logged(Session *session, uint64_t state_seq, bool open, ve dout(20) << " killing client lease of " << *dn << dendl; dn->remove_client_lease(r, mds->locker); } - if (client_reconnect_gather.count(session->info.get_client())) { + if (client_reconnect_gather.erase(session->info.get_client())) { dout(20) << " removing client from reconnect set" << dendl; - client_reconnect_gather.erase(session->info.get_client()); - if (client_reconnect_gather.empty()) { dout(7) << " client " << session->info.inst << " was last reconnect, finishing" << dendl; reconnect_gather_finish(); } } + if (client_reclaim_gather.erase(session->info.get_client())) { + dout(20) << " removing client from reclaim set" << dendl; + if (client_reclaim_gather.empty()) { + dout(7) << " client " << session->info.inst << " was last reclaimed, finishing" << dendl; + mds->maybe_clientreplay_done(); + } + } if (session->is_closing()) { // mark con disposable. if there is a fault, we will get a @@ -1118,11 +1123,14 @@ void Server::reconnect_clients(MDSInternalContext *reconnect_done_) { reconnect_done = reconnect_done_; + auto now = clock::now(); set sessions; mds->sessionmap.get_client_session_set(sessions); for (auto session : sessions) { - if (session->is_open()) - client_reconnect_gather.insert(session->get_client()); + if (session->is_open()) { + client_reconnect_gather.insert(session->get_client()); + session->last_cap_renew = now; + } } if (client_reconnect_gather.empty()) { @@ -1383,7 +1391,18 @@ void Server::reconnect_tick() ++p) { Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->v)); ceph_assert(session); - dout(1) << "reconnect gave up on " << session->info.inst << dendl; + + // Keep sessions that have specified timeout. These sessions will prevent + // mds from going to active. MDS goes to active after they all have been + // killed or reclaimed. + if (session->info.client_metadata.find("timeout") != + session->info.client_metadata.end()) { + dout(1) << "reconnect keeps " << session->info.inst << ", need to be reclaimed" << dendl; + client_reclaim_gather.insert(session->get_client()); + continue; + } + + dout(1) << "reconnect gives up on " << session->info.inst << dendl; mds->clog->warn() << "evicting unresponsive client " << *session << ", after waiting " << g_conf()->mds_reconnect_timeout diff --git a/src/mds/Server.h b/src/mds/Server.h index 40b1077d2004..3c865baea5b7 100644 --- a/src/mds/Server.h +++ b/src/mds/Server.h @@ -145,6 +145,8 @@ public: size_t apply_blacklist(const std::set &blacklist); void journal_close_session(Session *session, int state, Context *on_safe); + set client_reclaim_gather; + size_t get_num_pending_reclaim() const { return client_reclaim_gather.size(); } Session *find_session_by_uuid(std::string_view uuid); void reclaim_session(Session *session, const MClientReclaim::const_ref &m); void finish_reclaim_session(Session *session, const MClientReclaimReply::ref &reply=nullptr);