mds: limit maximum number of caps held by session

This is to prevent unsustainable situations where a client has so many
outstanding caps that a linear traversal/operation on the session's caps takes
unacceptable amounts of time.

Fixes: http://tracker.ceph.com/issues/38022
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index c9b869a3602361d500a08418d8f940b1d82fde55..e04ce58793165c979a9621147335cbd430f168b2 100644 (file)
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -173,6 +173,12 @@
   to not attempt recalling too many caps at once, leading to instability.
   MDS with a large cache (64GB+) should be more stable.
 
+* MDS now provides a config option "mds_max_caps_per_client" (default: 1M) to
+  limit the number of caps a client session may hold. Long running client
+  sessions with a large number of caps have been a source of instability in the
+  MDS when all of these caps need to be processed during certain session
+  events. It is recommended to not unnecessarily increase this value.
+
 >=13.1.0
 --------
 

diff --git a/src/common/options.cc b/src/common/options.cc
index 090e126fa788f2eb0e18c1e220f8223ee4ac7ff6..547b8b8757a333443f29004c50a78c46d1852bd6 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7629,6 +7629,10 @@ std::vector<Option> get_mds_options() {
     .set_default(100)
     .set_description("minimum number of capabilities a client may hold"),
 
+    Option("mds_max_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(1_M)
+    .set_description("maximum number of capabilities a client may hold"),
+
     Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
      .set_default(0)
      .set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 76539437544657143ec38fc5a1c29cc085edfe45..e9086e6e85c57f7738a7a0a2b580b88a7ae04737 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -715,6 +715,7 @@ void MDSRankDispatcher::tick()
   sessionmap.update_average_session_age();
 
   if (is_active() || is_stopping()) {
+    server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
     mdcache->trim();
     mdcache->trim_client_leases();
     mdcache->check_memory_usage();

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 146cb39ea1a16832339f2bd8482410c32b961288..6f2029332346e0c985155e6808b3936ef1226805 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1525,18 +1525,28 @@ std::pair<bool, uint64_t> Server::recall_client_state(MDSGatherBuilder* gather,
 {
   const auto now = clock::now();
   const bool steady = flags&RecallFlags::STEADY;
+  const bool enforce_max = flags&RecallFlags::ENFORCE_MAX;
 
+  const auto max_caps_per_client = g_conf().get_val<uint64_t>("mds_max_caps_per_client");
   const auto min_caps_per_client = g_conf().get_val<uint64_t>("mds_min_caps_per_client");
   const auto recall_global_max_decay_threshold = g_conf().get_val<Option::size_t>("mds_recall_global_max_decay_threshold");
   const auto recall_max_caps = g_conf().get_val<Option::size_t>("mds_recall_max_caps");
   const auto recall_max_decay_threshold = g_conf().get_val<Option::size_t>("mds_recall_max_decay_threshold");
 
-  dout(10) << __func__ << ": caps per client " << min_caps_per_client << "/" << Capability::count() << dendl;
+  dout(10) << __func__ << ":"
+           << " min=" << min_caps_per_client
+           << " max=" << max_caps_per_client
+           << " total=" << Capability::count()
+           << " flags=0x" << std::hex << flags
+           << dendl;
 
   /* trim caps of sessions with the most caps first */
   std::multimap<uint64_t, Session*> caps_session;
-  auto f = [&caps_session](auto& s) {
-    caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(s->caps.size()), std::forward_as_tuple(s));
+  auto f = [&caps_session, enforce_max, max_caps_per_client](auto& s) {
+    auto num_caps = s->caps.size();
+    if (!enforce_max || num_caps > max_caps_per_client) {
+      caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(num_caps), std::forward_as_tuple(s));
+    }
   };
   mds->sessionmap.get_client_sessions(std::move(f));
 

diff --git a/src/mds/Server.h b/src/mds/Server.h
index a9a8f09ec3d99a19d0e2324f47733875ffe8e6fa..fc504e3249566adfba649bc0625516fe34cf7589 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -172,6 +172,7 @@ public:
   enum RecallFlags {
     NONE = 0,
     STEADY = (1<<0),
+    ENFORCE_MAX = (1<<1),
   };
   std::pair<bool, uint64_t> recall_client_state(MDSGatherBuilder* gather, enum RecallFlags=RecallFlags::NONE);
   void force_clients_readonly();