From 4b72cc60ebce4bd43eccb00f883411e85b0e3704 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 7 Aug 2019 10:35:02 -0700
Subject: [PATCH] mds: trim cache on regular schedule

Do this outside the standard tick interval as it needs to be driven more
frequently to keep up with client workloads that generate a lot of
capabilities.

Fixes: https://tracker.ceph.com/issues/41141
Fixes: https://tracker.ceph.com/issues/41140
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/options.cc |  4 ++++
 src/mds/MDCache.cc    | 37 +++++++++++++++++++++++++++++++++++++
 src/mds/MDCache.h     |  9 +++++++++
 src/mds/MDSRank.cc    |  8 +-------
 src/mds/MDSRank.h     |  4 ++++
 5 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index a819ed38335..c90b3296c56 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7682,6 +7682,10 @@ std::vector<Option> get_mds_options() {
     .set_default(64_K)
     .set_description("maximum aggregate size of extended attributes on a file"),
 
+    Option("mds_cache_trim_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
+    .set_default(1)
+    .set_description("interval in seconds between cache trimming"),
+
     Option("mds_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description("maximum number of inodes in MDS cache (<=0 is unlimited)")
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index de4bef951b1..4d854fa7be2 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -157,6 +157,36 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
   bottom_lru.lru_set_midpoint(0);
 
   decayrate.set_halflife(g_conf()->mds_decay_halflife);
+
+  upkeeper = std::thread([this]() {
+    std::unique_lock lock(upkeep_mutex);
+    while (!upkeep_trim_shutdown.load()) {
+      auto now = clock::now();
+      auto since = now-upkeep_last_trim;
+      auto interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
+      if (since >= interval*.90) {
+        lock.unlock(); /* mds_lock -> upkeep_mutex */
+        std::scoped_lock mds_lock(mds->mds_lock);
+        lock.lock();
+        if (upkeep_trim_shutdown.load())
+          return;
+        if (mds->is_cache_trimmable()) {
+          dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl;
+          trim_client_leases();
+          trim();
+          check_memory_usage();
+          mds->server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
+          upkeep_last_trim = clock::now();
+        } else {
+          dout(10) << "cache not ready for trimming" << dendl;
+        }
+      } else {
+        interval -= since;
+      }
+      dout(20) << "upkeep thread waiting interval " << interval << dendl;
+      upkeep_cvar.wait_for(lock, interval);
+    }
+  });
 }
 
 MDCache::~MDCache() 
@@ -164,6 +194,8 @@ MDCache::~MDCache()
   if (logger) {
     g_ceph_context->get_perfcounters_collection()->remove(logger.get());
   }
+  if (upkeeper.joinable())
+    upkeeper.join();
 }
 
 void MDCache::handle_conf_change(const std::set<std::string>& changed, const MDSMap& mdsmap)
@@ -203,6 +235,11 @@ void MDCache::log_stat()
 
 bool MDCache::shutdown()
 {
+  {
+    std::scoped_lock lock(upkeep_mutex);
+    upkeep_trim_shutdown = true;
+    upkeep_cvar.notify_one();
+  }
   if (lru.lru_get_size() > 0) {
     dout(7) << "WARNING: mdcache shutdown with non-empty cache" << dendl;
     //show_cache();
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index c3c65e37cf0..8dec2a37241 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -17,7 +17,9 @@
 #ifndef CEPH_MDCACHE_H
 #define CEPH_MDCACHE_H
 
+#include <atomic>
 #include <string_view>
+#include <thread>
 
 #include "common/DecayCounter.h"
 #include "include/types.h"
@@ -1321,6 +1323,13 @@ public:
   std::set<CInode *> export_pin_delayed_queue;
 
   OpenFileTable open_file_table;
+
+private:
+  std::thread upkeeper;
+  ceph::mutex upkeep_mutex = ceph::make_mutex("MDCache::upkeep_mutex");
+  ceph::condition_variable upkeep_cvar;
+  time upkeep_last_trim = time::min();
+  std::atomic<bool> upkeep_trim_shutdown{false};
 };
 
 class C_MDS_RetryRequest : public MDSInternalContext {
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 46112b9771f..46b741439c6 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -746,13 +746,7 @@ void MDSRankDispatcher::tick()
   }
 
   // ...
-  if (is_clientreplay() || is_active() || is_stopping()) {
-    mdcache->trim_client_leases();
-    mdcache->trim();
-    mdcache->check_memory_usage();
-
-    server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
-
+  if (is_cache_trimmable()) {
     server->find_idle_sessions();
     server->evict_cap_revoke_non_responders();
     locker->tick();
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 1ae2c6a3bc6..eff0d872303 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -230,6 +230,10 @@ class MDSRank {
     bool is_cluster_degraded() const { return cluster_degraded; }
     bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); }
 
+    bool is_cache_trimmable() const {
+      return is_clientreplay() || is_active() || is_stopping();
+    }
+
     void handle_write_error(int err);
 
     void update_mlogger();
-- 
2.39.5