mds: do not evict client on laggy osds

A client might get unresponsive/laggy due to laggy OSD(s).
This change provides us a way to defer client eviction in
such scenarios

also adds helpers:
- get_laggy_clients()
- clear_laggy_clients()

and call clear_laggy_clients() before calling related
Server methods

Fixes: https://tracker.ceph.com/issues/58023
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 6b8b24d808e0dc45b5c002134dc8e64602f4d30e..6dafd9b70fd1c647778eb880063fb9169a3e5750 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -743,6 +743,7 @@ void MDSRankDispatcher::tick()
 
   // ...
   if (is_clientreplay() || is_active() || is_stopping()) {
+    server->clear_laggy_clients();
     server->find_idle_sessions();
     server->evict_cap_revoke_non_responders();
     locker->tick();

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index d7ae5836c9ade547a15bce896753e8ee182e9593..1b0dca25b22b8deb02c95dbda58c80bf22f6d194 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1132,10 +1132,12 @@ void Server::find_idle_sessions()
     return;
   }
 
-  std::vector<Session*> to_evict;
-
   bool defer_session_stale = g_conf().get_val<bool>("mds_defer_session_stale");
   const auto sessions_p1 = mds->sessionmap.by_state.find(Session::STATE_OPEN);
+  bool defer_client_eviction =
+  g_conf().get_val<bool>("defer_client_eviction_on_laggy_osds")
+  && mds->objecter->with_osdmap([](const OSDMap &map) {
+    return map.any_osd_laggy(); });
   if (sessions_p1 != mds->sessionmap.by_state.end() && !sessions_p1->second->empty()) {
     std::vector<Session*> new_stale;
 
@@ -1160,7 +1162,7 @@ void Server::find_idle_sessions()
        dout(20) << "evicting session " << session->info.inst << " since autoclose "
                    "has arrived" << dendl;
        // evict session without marking it stale
-       to_evict.push_back(session);
+       laggy_clients.insert(session->get_client());
        continue;
       }
 
@@ -1189,7 +1191,7 @@ void Server::find_idle_sessions()
        }
 
        // do not go through stale, evict it directly.
-       to_evict.push_back(session);
+       laggy_clients.insert(session->get_client());
       } else {
        dout(10) << "new stale session " << session->info.inst
                 << " last renewed caps " << last_cap_renew_span << "s ago" << dendl;
@@ -1205,7 +1207,7 @@ void Server::find_idle_sessions()
        auto m = make_message<MClientSession>(CEPH_SESSION_STALE);
        mds->send_message_client(m, session);
       } else {
-       to_evict.push_back(session);
+       laggy_clients.insert(session->get_client());
       }
     }
   }
@@ -1224,11 +1226,21 @@ void Server::find_idle_sessions()
                 << " and recently renewed caps " << last_cap_renew_span << "s ago" << dendl;
        break;
       }
-      to_evict.push_back(session);
+      laggy_clients.insert(session->get_client());
     }
   }
 
-  for (auto session: to_evict) {
+  // don't evict client(s) if osds are laggy
+  if(defer_client_eviction && !laggy_clients.empty()) {
+    dout(5) << "Detected " << laggy_clients.size()
+            << " laggy clients, possibly due to laggy OSDs."
+               " Eviction is skipped until the OSDs return to normal."
+            << dendl;
+    return;
+  }
+
+  for (auto client: laggy_clients) {
+    Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(client.v));
     if (session->is_importing()) {
       dout(10) << "skipping session " << session->info.inst << ", it's being imported" << dendl;
       continue;
@@ -1255,6 +1267,20 @@ void Server::evict_cap_revoke_non_responders() {
   }
 
   auto&& to_evict = mds->locker->get_late_revoking_clients(cap_revoke_eviction_timeout);
+  // don't evict client(s) if osds are laggy
+  bool defer_client_eviction =
+  g_conf().get_val<bool>("defer_client_eviction_on_laggy_osds")
+  && mds->objecter->with_osdmap([](const OSDMap &map) {
+    return map.any_osd_laggy(); })
+  && to_evict.size();
+  if(defer_client_eviction) {
+    laggy_clients.insert(to_evict.begin(), to_evict.end());
+    dout(0) << "Detected " << to_evict.size()
+            << " unresponsive clients, possibly due to laggy OSDs."
+               " Eviction is skipped until the OSDs return to normal."
+            << dendl;
+    return;
+  }
 
   for (auto const &client: to_evict) {
     mds->clog->warn() << "client id " << client << " has not responded to"

diff --git a/src/mds/Server.h b/src/mds/Server.h
index ba38ccb2db4e680d46bc36bf2ca220b18801be26..0d6287e2ccf4ecac73cd0f4a35331782658f06e5 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -330,6 +330,13 @@ public:
 
   std::set<client_t> client_reclaim_gather;
 
+  std::set<client_t> get_laggy_clients() const {
+    return laggy_clients;
+  }
+  void clear_laggy_clients() {
+    laggy_clients.clear();
+  }
+
   const bufferlist& get_snap_trace(Session *session, SnapRealm *realm) const;
   const bufferlist& get_snap_trace(client_t client, SnapRealm *realm) const;
 
@@ -555,6 +562,9 @@ private:
 
   size_t alternate_name_max = g_conf().get_val<Option::size_t>("mds_alternate_name_max");
   size_t fscrypt_last_block_max_size = g_conf().get_val<Option::size_t>("mds_fscrypt_last_block_max_size");
+
+  // record laggy clients due to laggy OSDs
+  std::set<client_t> laggy_clients;
 };
 
 static inline constexpr auto operator|(Server::RecallFlags a, Server::RecallFlags b) {