mds: limit maximum number of caps held by session

This is to prevent unsustainable situations where a client has so many
outstanding caps that a linear traversal/operation on the session's caps takes
unacceptable amounts of time.

Fixes: http://tracker.ceph.com/issues/38022
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 48ca097)

Conflicts:
	PendingReleaseNotes
	src/mds/Server.cc

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 75338bdae1f03c33fa67e41bfb98c6c782c897d1..142ac77d659fbcdfbfa7b06c51dfe6937d567331 100644 (file)
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -9,6 +9,12 @@
   to not attempt recalling too many caps at once, leading to instability.
   MDS with a large cache (64GB+) should be more stable.
 
+* MDS now provides a config option "mds_max_caps_per_client" (default: 1M) to
+  limit the number of caps a client session may hold. Long running client
+  sessions with a large number of caps have been a source of instability in the
+  MDS when all of these caps need to be processed during certain session
+  events. It is recommended to not unnecessarily increase this value.
+
 * The `cache drop` admin socket command has been removed. The `ceph tell mds.X
   cache drop` remains.
 

diff --git a/src/common/options.cc b/src/common/options.cc
index ca0a0ebcaf8eeb68c349ec723d870e47185e3ef7..5a8ec3484aeb263d144aad793b962849555937de 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7179,6 +7179,10 @@ std::vector<Option> get_mds_options() {
     .set_default(100)
     .set_description("minimum number of capabilities a client may hold"),
 
+    Option("mds_max_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(1_M)
+    .set_description("maximum number of capabilities a client may hold"),
+
     Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
      .set_default(0)
      .set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 1710832a7c882c597ced80b251fa6eefede6a553..c89b486ecba121b0482a35ff2b49976bcf046f37 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -717,6 +717,7 @@ void MDSRankDispatcher::tick()
   sessionmap.update_average_session_age();
 
   if (is_active() || is_stopping()) {
+    server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
     mdcache->trim();
     mdcache->trim_client_leases();
     mdcache->check_memory_usage();

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 21c019e87bb85c3d79af76814b0ae30c18248d6c..9abed22544f698eae7263066dd336fb7e90d52d3 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1366,18 +1366,28 @@ std::pair<bool, uint64_t> Server::recall_client_state(MDSGatherBuilder* gather,
 {
   const auto now = clock::now();
   const bool steady = flags&RecallFlags::STEADY;
+  const bool enforce_max = flags&RecallFlags::ENFORCE_MAX;
 
+  const auto max_caps_per_client = g_conf->get_val<uint64_t>("mds_max_caps_per_client");
   const auto min_caps_per_client = g_conf->get_val<uint64_t>("mds_min_caps_per_client");
   const auto recall_global_max_decay_threshold = g_conf->get_val<Option::size_t>("mds_recall_global_max_decay_threshold");
   const auto recall_max_caps = g_conf->get_val<Option::size_t>("mds_recall_max_caps");
   const auto recall_max_decay_threshold = g_conf->get_val<Option::size_t>("mds_recall_max_decay_threshold");
 
-  dout(10) << __func__ << ": caps per client " << min_caps_per_client << "/" << Capability::count() << dendl;
+  dout(10) << __func__ << ":"
+           << " min=" << min_caps_per_client
+           << " max=" << max_caps_per_client
+           << " total=" << Capability::count()
+           << " flags=0x" << std::hex << flags
+           << dendl;
 
   /* trim caps of sessions with the most caps first */
   std::multimap<uint64_t, Session*> caps_session;
-  auto f = [&caps_session](auto& s) {
-    caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(s->caps.size()), std::forward_as_tuple(s));
+  auto f = [&caps_session, enforce_max, max_caps_per_client](auto& s) {
+    auto num_caps = s->caps.size();
+    if (!enforce_max || num_caps > max_caps_per_client) {
+      caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(num_caps), std::forward_as_tuple(s));
+    }
   };
   mds->sessionmap.get_client_sessions(std::move(f));
 

diff --git a/src/mds/Server.h b/src/mds/Server.h
index 6991fbd935ade279c6c259436f9db773318d0577..85761b55be88b4a58a3ec97f100804294bcf5dfa 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -157,6 +157,7 @@ public:
   enum RecallFlags {
     NONE = 0,
     STEADY = (1<<0),
+    ENFORCE_MAX = (1<<1),
   };
   std::pair<bool, uint64_t> recall_client_state(MDSGatherBuilder* gather, enum RecallFlags=RecallFlags::NONE);
   void force_clients_readonly();