mds: release free heap pages after trim

MDS free heap space can grow to large for some workloads (like smallfile
and recursive deletes). This can cause the MDS mapped memory to grow
well beyond memory targets.

When we finally use the PriorityCache in the MDS, this will not be
necessary anymore as the PriorityCache already does this.

Fixes: https://tracker.ceph.com/issues/42938
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit af4cac5ec7bab4e5bf8936cd685f6ee8dcb38127)

Conflicts:
    src/mds/MDCache.cc

diff --git a/src/common/options.cc b/src/common/options.cc
index cc10924dc51163db73d29d1bd35aba8bd4a1d03c..02a909e57565d183246d4857bd3104e02a1a1696 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7621,7 +7621,13 @@ std::vector<Option> get_mds_options() {
 
     Option("mds_cache_trim_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
     .set_default(1)
-    .set_description("interval in seconds between cache trimming"),
+    .set_description("interval in seconds between cache trimming")
+    .set_flag(Option::FLAG_RUNTIME),
+
+    Option("mds_cache_release_free_interval", Option::TYPE_SECS, Option::LEVEL_DEV)
+    .set_default(10)
+    .set_description("interval in seconds between heap releases")
+    .set_flag(Option::FLAG_RUNTIME),
 
     Option("mds_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index e9238391cccfcbf70b28f34cdda82aaefc6ce6c2..5538e70e9d6cc0ea1a675ccd33715b31270a4d68 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -163,8 +163,8 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
     while (!upkeep_trim_shutdown.load()) {
       auto now = clock::now();
       auto since = now-upkeep_last_trim;
-      auto interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
-      if (since >= interval*.90) {
+      auto trim_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
+      if (since >= trim_interval*.90) {
         lock.unlock(); /* mds_lock -> upkeep_mutex */
         std::scoped_lock mds_lock(mds->mds_lock);
         lock.lock();
@@ -177,12 +177,24 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
           check_memory_usage();
           mds->server->recall_client_state(nullptr, Server::RecallFlags::ENFORCE_MAX);
           upkeep_last_trim = clock::now();
+          upkeep_last_trim = now = clock::now();
         } else {
           dout(10) << "cache not ready for trimming" << dendl;
         }
       } else {
-        interval -= since;
+        trim_interval -= since;
+      }
+      since = now-upkeep_last_release;
+      auto release_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_release_free_interval"));
+      if (since >= release_interval) {
+        /* XXX not necessary once MDCache uses PriorityCache */
+        dout(10) << "releasing free memory" << dendl;
+        ceph_heap_release_free_memory();
+        upkeep_last_release = clock::now();
+      } else {
+        release_interval -= since;
       }
+      auto interval = std::min(release_interval, trim_interval);
       dout(20) << "upkeep thread waiting interval " << interval << dendl;
       upkeep_cvar.wait_for(lock, interval);
     }

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 9cd83419d9e3fe5e47182b42ce8502709dad3cc9..1b37013d143037fb6c1a1e7df1d8a21f8a98545f 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1331,6 +1331,7 @@ private:
   ceph::mutex upkeep_mutex = ceph::make_mutex("MDCache::upkeep_mutex");
   ceph::condition_variable upkeep_cvar;
   time upkeep_last_trim = time::min();
+  time upkeep_last_release = time::min();
   std::atomic<bool> upkeep_trim_shutdown{false};
 };