mds: limit maximum number of caps held by session

This is to prevent unsustainable situations where a client has so many
outstanding caps that a linear traversal/operation on the session's caps takes
unacceptable amounts of time.

Fixes: http://tracker.ceph.com/issues/38022
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 48ca097a9f537545b4b0c864ebb8c5e81d821098)

Conflicts:
	PendingReleaseNotes
	src/mds/Server.cc

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 1218bef68e7fe37fa3fa47fb4ebedfc71db40c98..839961979acc57e50afc79f438649486b007db37 100644 (file)
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -20,6 +20,12 @@
   to not attempt recalling too many caps at once, leading to instability.
   MDS with a large cache (64GB+) should be more stable.
 
+* MDS now provides a config option "mds_max_caps_per_client" (default: 1M) to
+  limit the number of caps a client session may hold. Long running client
+  sessions with a large number of caps have been a source of instability in the
+  MDS when all of these caps need to be processed during certain session
+  events. It is recommended to not unnecessarily increase this value.
+
 >= 12.1.2
 ---------
 * When running 'df' on a CephFS filesystem comprising exactly one data pool,

diff --git a/src/common/options.cc b/src/common/options.cc
index 3497b06a1fa7f0d7db6badecc85c25bd86533031..40efed5ac0ca38b2597ff7d3cec1e841af3a4e00 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6542,6 +6542,10 @@ std::vector<Option> get_mds_options() {
     .set_default(100)
     .set_description("minimum number of capabilities a client may hold"),
 
+    Option("mds_max_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(1_M)
+    .set_description("maximum number of capabilities a client may hold"),
+
     Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
      .set_default(0)
      .set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 898043fd0d95d9a099ec4b552de094cc6420638c..24416aa20d2b38d16cad6bc5a3d9db91b5f3c08e 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -674,6 +674,7 @@ void MDSRankDispatcher::tick()
   sessionmap.update_average_session_age();
 
   if (is_active() || is_stopping()) {
+    server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
     mdcache->trim();
     mdcache->trim_client_leases();
     mdcache->check_memory_usage();

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 315aa4db97c4ca420e65f0a867a535c456e03230..dea4a35a5bb6129441c914709afb1fbad47cc42d 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1231,18 +1231,28 @@ std::pair<bool, uint64_t> Server::recall_client_state(MDSGatherBuilder* gather,
 {
   const auto now = clock::now();
   const bool steady = flags&RecallFlags::STEADY;
+  const bool enforce_max = flags&RecallFlags::ENFORCE_MAX;
 
+  const auto max_caps_per_client = g_conf->get_val<uint64_t>("mds_max_caps_per_client");
   const auto min_caps_per_client = g_conf->get_val<uint64_t>("mds_min_caps_per_client");
   const auto recall_global_max_decay_threshold = g_conf->get_val<uint64_t>("mds_recall_global_max_decay_threshold");
   const auto recall_max_caps = g_conf->get_val<uint64_t>("mds_recall_max_caps");
   const auto recall_max_decay_threshold = g_conf->get_val<uint64_t>("mds_recall_max_decay_threshold");
 
-  dout(10) << __func__ << ": caps per client " << min_caps_per_client << "/" << Capability::count() << dendl;
+  dout(10) << __func__ << ":"
+           << " min=" << min_caps_per_client
+           << " max=" << max_caps_per_client
+           << " total=" << Capability::count()
+           << " flags=0x" << std::hex << flags
+           << dendl;
 
   /* trim caps of sessions with the most caps first */
   std::multimap<uint64_t, Session*> caps_session;
-  auto f = [&caps_session](auto& s) {
-    caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(s->caps.size()), std::forward_as_tuple(s));
+  auto f = [&caps_session, enforce_max, max_caps_per_client](Session* s) {
+    auto num_caps = s->caps.size();
+    if (!enforce_max || num_caps > max_caps_per_client) {
+      caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(num_caps), std::forward_as_tuple(s));
+    }
   };
   mds->sessionmap.get_client_sessions(std::move(f));
 

diff --git a/src/mds/Server.h b/src/mds/Server.h
index ab16a55bca24c73bdd7ebe5cd5de62c422d2a2b4..c55701f20fb5934e4f2d81cd8273a3a015a86091 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -149,6 +149,7 @@ public:
   enum RecallFlags {
     NONE = 0,
     STEADY = (1<<0),
+    ENFORCE_MAX = (1<<1),
   };
   std::pair<bool, uint64_t> recall_client_state(MDSGatherBuilder* gather, enum RecallFlags=RecallFlags::NONE);
   void force_clients_readonly();