mds: reclaim session before allowing mds to become active

Recovering MDS does not go to active state when there are sessions
that need to be reclaimed. To make mds go to active state, Other
client should kill these session by calling:
  ceph_start_reclaim(cmount, uuid, CEPH_RECLAIM_RESET)

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 58ed76055b307019249889f523fd7a86e8427ce1..22c779c43175d82f9b3c8acfb45636ea8081c796 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -329,6 +329,8 @@ void MDSRankDispatcher::tick()
 
   if (is_reconnect())
     server->reconnect_tick();
+  if (is_clientreplay())
+    maybe_clientreplay_done();
 
   if (is_active()) {
     balancer->tick();
@@ -618,17 +620,6 @@ bool MDSRank::_dispatch(const Message::const_ref &m, bool new_msg)
     return true;
   }
 
-  // done with all client replayed requests?
-  if (is_clientreplay() &&
-      mdcache->is_open() &&
-      replay_queue.empty() &&
-      beacon.get_want_state() == MDSMap::STATE_CLIENTREPLAY) {
-    int num_requests = mdcache->get_num_client_requests();
-    dout(10) << " still have " << num_requests << " active replay requests" << dendl;
-    if (num_requests == 0)
-      clientreplay_done();
-  }
-
   // hack: thrash exports
   static utime_t start;
   utime_t now = ceph_clock_now();
@@ -1481,7 +1472,7 @@ void MDSRank::rejoin_done()
     return;
   }
 
-  if (replay_queue.empty())
+  if (replay_queue.empty() && !server->get_num_pending_reclaim())
     request_state(MDSMap::STATE_ACTIVE);
   else
     request_state(MDSMap::STATE_CLIENTREPLAY);
@@ -1497,13 +1488,31 @@ void MDSRank::clientreplay_start()
 
 bool MDSRank::queue_one_replay()
 {
-  if (replay_queue.empty()) {
+  if (!replay_queue.empty()) {
+    queue_waiter(replay_queue.front());
+    replay_queue.pop_front();
+    return true;
+  }
+  // don't go to active if there are session waiting for being reclaimed
+  if (!server->get_num_pending_reclaim())
     mdlog->wait_for_safe(new C_MDS_VoidFn(this, &MDSRank::clientreplay_done));
-    return false;
+  return false;
+}
+
+void MDSRank::maybe_clientreplay_done()
+{
+  if (is_clientreplay() &&
+      get_want_state() == MDSMap::STATE_CLIENTREPLAY &&
+      replay_queue.empty()) {
+    int num_requests = mdcache->get_num_client_requests();
+    int num_reclaim = server->get_num_pending_reclaim();
+    if (!num_requests && !num_reclaim) {
+      clientreplay_done();
+    } else {
+      dout(1) << " still have " << num_requests << " active replay requests, "
+             << num_reclaim << " sessions need to be reclaimed" << dendl;
+    }
   }
-  queue_waiter(replay_queue.front());
-  replay_queue.pop_front();
-  return true;
 }
 
 void MDSRank::clientreplay_done()

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 69a1fb9269f1b9b3656c40052c5d2ef72ecf8f17..9055f39c2959b0bcc0594814188d14df3bf12c29 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -421,6 +421,7 @@ class MDSRank {
     }
 
     bool queue_one_replay();
+    void maybe_clientreplay_done();
 
     void set_osd_epoch_barrier(epoch_t e);
     epoch_t get_osd_epoch_barrier() const {return osd_epoch_barrier;}

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 956a2adbe65419a48c33d8a7dabc2c27d6532815..9dbd5924235f0c388b3b05d1d56f31525277b752 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -704,15 +704,20 @@ void Server::_session_logged(Session *session, uint64_t state_seq, bool open, ve
       dout(20) << " killing client lease of " << *dn << dendl;
       dn->remove_client_lease(r, mds->locker);
     }
-    if (client_reconnect_gather.count(session->info.get_client())) {
+    if (client_reconnect_gather.erase(session->info.get_client())) {
       dout(20) << " removing client from reconnect set" << dendl;
-      client_reconnect_gather.erase(session->info.get_client());
-
       if (client_reconnect_gather.empty()) {
         dout(7) << " client " << session->info.inst << " was last reconnect, finishing" << dendl;
         reconnect_gather_finish();
       }
     }
+    if (client_reclaim_gather.erase(session->info.get_client())) {
+      dout(20) << " removing client from reclaim set" << dendl;
+      if (client_reclaim_gather.empty()) {
+        dout(7) << " client " << session->info.inst << " was last reclaimed, finishing" << dendl;
+       mds->maybe_clientreplay_done();
+      }
+    }
     
     if (session->is_closing()) {
       // mark con disposable.  if there is a fault, we will get a
@@ -1118,11 +1123,14 @@ void Server::reconnect_clients(MDSInternalContext *reconnect_done_)
 {
   reconnect_done = reconnect_done_;
 
+  auto now = clock::now();
   set<Session*> sessions;
   mds->sessionmap.get_client_session_set(sessions);
   for (auto session : sessions) {
-    if (session->is_open())
-       client_reconnect_gather.insert(session->get_client());
+    if (session->is_open()) {
+      client_reconnect_gather.insert(session->get_client());
+      session->last_cap_renew = now;
+    }
   }
 
   if (client_reconnect_gather.empty()) {
@@ -1383,7 +1391,18 @@ void Server::reconnect_tick()
         ++p) {
       Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->v));
       ceph_assert(session);
-      dout(1) << "reconnect gave up on " << session->info.inst << dendl;
+
+      // Keep sessions that have specified timeout. These sessions will prevent
+      // mds from going to active. MDS goes to active after they all have been
+      // killed or reclaimed.
+      if (session->info.client_metadata.find("timeout") !=
+         session->info.client_metadata.end()) {
+       dout(1) << "reconnect keeps " << session->info.inst << ", need to be reclaimed" << dendl;
+       client_reclaim_gather.insert(session->get_client());
+       continue;
+      }
+
+      dout(1) << "reconnect gives up on " << session->info.inst << dendl;
 
       mds->clog->warn() << "evicting unresponsive client " << *session
                         << ", after waiting " << g_conf()->mds_reconnect_timeout

diff --git a/src/mds/Server.h b/src/mds/Server.h
index 40b1077d2004bd9f41ad843062caa244f29234f4..3c865baea5b759195a0596f7d95eb4bd9e54ea4d 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -145,6 +145,8 @@ public:
   size_t apply_blacklist(const std::set<entity_addr_t> &blacklist);
   void journal_close_session(Session *session, int state, Context *on_safe);
 
+  set<client_t> client_reclaim_gather;
+  size_t get_num_pending_reclaim() const { return client_reclaim_gather.size(); }
   Session *find_session_by_uuid(std::string_view uuid);
   void reclaim_session(Session *session, const MClientReclaim::const_ref &m);
   void finish_reclaim_session(Session *session, const MClientReclaimReply::ref &reply=nullptr);