From af4cac5ec7bab4e5bf8936cd685f6ee8dcb38127 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 21 Nov 2019 10:09:39 -0800
Subject: [PATCH] mds: release free heap pages after trim

MDS free heap space can grow to large for some workloads (like smallfile
and recursive deletes). This can cause the MDS mapped memory to grow
well beyond memory targets.

When we finally use the PriorityCache in the MDS, this will not be
necessary anymore as the PriorityCache already does this.

Fixes: https://tracker.ceph.com/issues/42938
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/options.cc |  8 +++++++-
 src/mds/MDCache.cc    | 19 +++++++++++++++----
 src/mds/MDCache.h     |  1 +
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index c5cad0169cfa..7f65b8bd048d 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7455,7 +7455,13 @@ std::vector<Option> get_mds_options() {
 
     Option("mds_cache_trim_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
     .set_default(1)
-    .set_description("interval in seconds between cache trimming"),
+    .set_description("interval in seconds between cache trimming")
+    .set_flag(Option::FLAG_RUNTIME),
+
+    Option("mds_cache_release_free_interval", Option::TYPE_SECS, Option::LEVEL_DEV)
+    .set_default(10)
+    .set_description("interval in seconds between heap releases")
+    .set_flag(Option::FLAG_RUNTIME),
 
     Option("mds_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index de35bd444476..2a9248ab5584 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -163,8 +163,8 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
     while (!upkeep_trim_shutdown.load()) {
       auto now = clock::now();
       auto since = now-upkeep_last_trim;
-      auto interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
-      if (since >= interval*.90) {
+      auto trim_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
+      if (since >= trim_interval*.90) {
         lock.unlock(); /* mds_lock -> upkeep_mutex */
         std::scoped_lock mds_lock(mds->mds_lock);
         lock.lock();
@@ -177,13 +177,24 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
           check_memory_usage();
           auto flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS;
           mds->server->recall_client_state(nullptr, flags);
-          upkeep_last_trim = clock::now();
+          upkeep_last_trim = now = clock::now();
         } else {
           dout(10) << "cache not ready for trimming" << dendl;
         }
       } else {
-        interval -= since;
+        trim_interval -= since;
+      }
+      since = now-upkeep_last_release;
+      auto release_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_release_free_interval"));
+      if (since >= release_interval) {
+        /* XXX not necessary once MDCache uses PriorityCache */
+        dout(10) << "releasing free memory" << dendl;
+        ceph_heap_release_free_memory();
+        upkeep_last_release = clock::now();
+      } else {
+        release_interval -= since;
       }
+      auto interval = std::min(release_interval, trim_interval);
       dout(20) << "upkeep thread waiting interval " << interval << dendl;
       upkeep_cvar.wait_for(lock, interval);
     }
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index abed6a6d26e3..d79cb8912f7a 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1286,6 +1286,7 @@ class MDCache {
   ceph::mutex upkeep_mutex = ceph::make_mutex("MDCache::upkeep_mutex");
   ceph::condition_variable upkeep_cvar;
   time upkeep_last_trim = time::min();
+  time upkeep_last_release = time::min();
   std::atomic<bool> upkeep_trim_shutdown{false};
 };
 
-- 
2.47.3