From 2c63330748c75f218fa36d8aba6a289a27cba893 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 24 Jan 2019 14:23:08 -0800
Subject: [PATCH] mds: limit maximum number of caps held by session

This is to prevent unsustainable situations where a client has so many
outstanding caps that a linear traversal/operation on the session's caps takes
unacceptable amounts of time.

Fixes: http://tracker.ceph.com/issues/38022
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 48ca097)

Conflicts:
	PendingReleaseNotes
	src/mds/Server.cc
---
 PendingReleaseNotes   |  6 ++++++
 src/common/options.cc |  4 ++++
 src/mds/MDSRank.cc    |  1 +
 src/mds/Server.cc     | 16 +++++++++++++---
 src/mds/Server.h      |  1 +
 5 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 75338bdae1f03..142ac77d659fb 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -9,6 +9,12 @@
   to not attempt recalling too many caps at once, leading to instability.
   MDS with a large cache (64GB+) should be more stable.
 
+* MDS now provides a config option "mds_max_caps_per_client" (default: 1M) to
+  limit the number of caps a client session may hold. Long running client
+  sessions with a large number of caps have been a source of instability in the
+  MDS when all of these caps need to be processed during certain session
+  events. It is recommended to not unnecessarily increase this value.
+
 * The `cache drop` admin socket command has been removed. The `ceph tell mds.X
   cache drop` remains.
 
diff --git a/src/common/options.cc b/src/common/options.cc
index ca0a0ebcaf8ee..5a8ec3484aeb2 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7179,6 +7179,10 @@ std::vector<Option> get_mds_options() {
     .set_default(100)
     .set_description("minimum number of capabilities a client may hold"),
 
+    Option("mds_max_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(1_M)
+    .set_description("maximum number of capabilities a client may hold"),
+
     Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
      .set_default(0)
      .set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 1710832a7c882..c89b486ecba12 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -717,6 +717,7 @@ void MDSRankDispatcher::tick()
   sessionmap.update_average_session_age();
 
   if (is_active() || is_stopping()) {
+    server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
     mdcache->trim();
     mdcache->trim_client_leases();
     mdcache->check_memory_usage();
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 21c019e87bb85..9abed22544f69 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1366,18 +1366,28 @@ std::pair<bool, uint64_t> Server::recall_client_state(MDSGatherBuilder* gather,
 {
   const auto now = clock::now();
   const bool steady = flags&RecallFlags::STEADY;
+  const bool enforce_max = flags&RecallFlags::ENFORCE_MAX;
 
+  const auto max_caps_per_client = g_conf->get_val<uint64_t>("mds_max_caps_per_client");
   const auto min_caps_per_client = g_conf->get_val<uint64_t>("mds_min_caps_per_client");
   const auto recall_global_max_decay_threshold = g_conf->get_val<Option::size_t>("mds_recall_global_max_decay_threshold");
   const auto recall_max_caps = g_conf->get_val<Option::size_t>("mds_recall_max_caps");
   const auto recall_max_decay_threshold = g_conf->get_val<Option::size_t>("mds_recall_max_decay_threshold");
 
-  dout(10) << __func__ << ": caps per client " << min_caps_per_client << "/" << Capability::count() << dendl;
+  dout(10) << __func__ << ":"
+           << " min=" << min_caps_per_client
+           << " max=" << max_caps_per_client
+           << " total=" << Capability::count()
+           << " flags=0x" << std::hex << flags
+           << dendl;
 
   /* trim caps of sessions with the most caps first */
   std::multimap<uint64_t, Session*> caps_session;
-  auto f = [&caps_session](auto& s) {
-    caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(s->caps.size()), std::forward_as_tuple(s));
+  auto f = [&caps_session, enforce_max, max_caps_per_client](auto& s) {
+    auto num_caps = s->caps.size();
+    if (!enforce_max || num_caps > max_caps_per_client) {
+      caps_session.emplace(std::piecewise_construct, std::forward_as_tuple(num_caps), std::forward_as_tuple(s));
+    }
   };
   mds->sessionmap.get_client_sessions(std::move(f));
 
diff --git a/src/mds/Server.h b/src/mds/Server.h
index 6991fbd935ade..85761b55be88b 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -157,6 +157,7 @@ public:
   enum RecallFlags {
     NONE = 0,
     STEADY = (1<<0),
+    ENFORCE_MAX = (1<<1),
   };
   std::pair<bool, uint64_t> recall_client_state(MDSGatherBuilder* gather, enum RecallFlags=RecallFlags::NONE);
   void force_clients_readonly();
-- 
2.39.5