mds: trim cache on regular schedule

Do this outside the standard tick interval as it needs to be driven more
frequently to keep up with client workloads that generate a lot of
capabilities.

Fixes: https://tracker.ceph.com/issues/41141
Fixes: https://tracker.ceph.com/issues/41140
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 4b72cc60ebce4bd43eccb00f883411e85b0e3704)

diff --git a/src/common/options.cc b/src/common/options.cc
index 4d42dfedc97882598ad481bb034f57776d451776..c70e2031fa57caac37957a1b3a285da7cc1c88ed 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7546,6 +7546,10 @@ std::vector<Option> get_mds_options() {
     .set_default(64_K)
     .set_description("maximum aggregate size of extended attributes on a file"),
 
+    Option("mds_cache_trim_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
+    .set_default(1)
+    .set_description("interval in seconds between cache trimming"),
+
     Option("mds_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description("maximum number of inodes in MDS cache (<=0 is unlimited)")

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 3009defa0ff965a27d260e6ad5db7e29383a6d89..fa60f7c112b6eb7691bf3920179879d5df5c0c25 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -157,6 +157,36 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
   bottom_lru.lru_set_midpoint(0);
 
   decayrate.set_halflife(g_conf()->mds_decay_halflife);
+
+  upkeeper = std::thread([this]() {
+    std::unique_lock lock(upkeep_mutex);
+    while (!upkeep_trim_shutdown.load()) {
+      auto now = clock::now();
+      auto since = now-upkeep_last_trim;
+      auto interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
+      if (since >= interval*.90) {
+        lock.unlock(); /* mds_lock -> upkeep_mutex */
+        std::scoped_lock mds_lock(mds->mds_lock);
+        lock.lock();
+        if (upkeep_trim_shutdown.load())
+          return;
+        if (mds->is_cache_trimmable()) {
+          dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl;
+          trim_client_leases();
+          trim();
+          check_memory_usage();
+          mds->server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
+          upkeep_last_trim = clock::now();
+        } else {
+          dout(10) << "cache not ready for trimming" << dendl;
+        }
+      } else {
+        interval -= since;
+      }
+      dout(20) << "upkeep thread waiting interval " << interval << dendl;
+      upkeep_cvar.wait_for(lock, interval);
+    }
+  });
 }
 
 MDCache::~MDCache() 
@@ -164,6 +194,8 @@ MDCache::~MDCache()
   if (logger) {
     g_ceph_context->get_perfcounters_collection()->remove(logger.get());
   }
+  if (upkeeper.joinable())
+    upkeeper.join();
 }
 
 void MDCache::handle_conf_change(const ConfigProxy& conf,
@@ -205,6 +237,11 @@ void MDCache::log_stat()
 
 bool MDCache::shutdown()
 {
+  {
+    std::scoped_lock lock(upkeep_mutex);
+    upkeep_trim_shutdown = true;
+    upkeep_cvar.notify_one();
+  }
   if (lru.lru_get_size() > 0) {
     dout(7) << "WARNING: mdcache shutdown with non-empty cache" << dendl;
     //show_cache();

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index ce68ca02c0fd40453d8be17a71fee69a99e77fb9..8d898e7f69a4d308942551ed83a25fc90c8885cb 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -17,7 +17,9 @@
 #ifndef CEPH_MDCACHE_H
 #define CEPH_MDCACHE_H
 
+#include <atomic>
 #include <string_view>
+#include <thread>
 
 #include "common/DecayCounter.h"
 #include "include/types.h"
@@ -1319,6 +1321,13 @@ public:
   std::set<CInode *> export_pin_queue;
 
   OpenFileTable open_file_table;
+
+private:
+  std::thread upkeeper;
+  ceph::mutex upkeep_mutex = ceph::make_mutex("MDCache::upkeep_mutex");
+  ceph::condition_variable upkeep_cvar;
+  time upkeep_last_trim = time::min();
+  std::atomic<bool> upkeep_trim_shutdown{false};
 };
 
 class C_MDS_RetryRequest : public MDSInternalContext {

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 8b622cab0f97171faa4c9e80296f7fc673a7b39f..2fcbb57e4a8e1c0ed7896517e90d8fa81ee3db50 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -742,13 +742,7 @@ void MDSRankDispatcher::tick()
   }
 
   // ...
-  if (is_clientreplay() || is_active() || is_stopping()) {
-    mdcache->trim_client_leases();
-    mdcache->trim();
-    mdcache->check_memory_usage();
-
-    server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
-
+  if (is_cache_trimmable()) {
     server->find_idle_sessions();
     server->evict_cap_revoke_non_responders();
     locker->tick();

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 4119c90d32d6b26abf5b51933978611e89865905..3200e05d85ef5e9954143f985ee6adb86d9a395e 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -229,6 +229,10 @@ class MDSRank {
     bool is_cluster_degraded() const { return cluster_degraded; }
     bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); }
 
+    bool is_cache_trimmable() const {
+      return is_clientreplay() || is_active() || is_stopping();
+    }
+
     void handle_write_error(int err);
 
     void handle_conf_change(const ConfigProxy& conf,