mds: trim cache regularly for standby-replay

This change is slightly awkward because standby-replay MDS do not do all
the kinds of upkeep a normal active MDS does. In particular, it is not
going to recall client state from clients.

This diff also merges the extra recall_client_state in
MDCache::check_memory_usage into its only caller (the upkeep thread)
where it was also doing a recall. That's just a matter of merging the
recall flags. This has the added benefit of making
MDCache::check_memory_usage callable for all MDS daemons regardless of
state.

Fixes: https://tracker.ceph.com/issues/50048
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 19293d9b9d19c32af4de655cd59e206056b2417d)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 842ad7d860190b2026c24729ed0eb058067cce4b..8236b429dc488f34866c745f5f393df0ef652cc7 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -162,47 +162,7 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
 
   decayrate.set_halflife(g_conf()->mds_decay_halflife);
 
-  upkeeper = std::thread([this]() {
-    std::unique_lock lock(upkeep_mutex);
-    while (!upkeep_trim_shutdown.load()) {
-      auto now = clock::now();
-      auto since = now-upkeep_last_trim;
-      auto trim_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
-      if (since >= trim_interval*.90) {
-        lock.unlock(); /* mds_lock -> upkeep_mutex */
-        std::scoped_lock mds_lock(mds->mds_lock);
-        lock.lock();
-        if (upkeep_trim_shutdown.load())
-          return;
-        if (mds->is_cache_trimmable()) {
-          dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl;
-          trim_client_leases();
-          trim();
-          check_memory_usage();
-          auto flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS;
-          mds->server->recall_client_state(nullptr, flags);
-          upkeep_last_trim = now = clock::now();
-        } else {
-          dout(10) << "cache not ready for trimming" << dendl;
-        }
-      } else {
-        trim_interval -= since;
-      }
-      since = now-upkeep_last_release;
-      auto release_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_release_free_interval"));
-      if (since >= release_interval*.90) {
-        /* XXX not necessary once MDCache uses PriorityCache */
-        dout(10) << "releasing free memory" << dendl;
-        ceph_heap_release_free_memory();
-        upkeep_last_release = clock::now();
-      } else {
-        release_interval -= since;
-      }
-      auto interval = std::min(release_interval, trim_interval);
-      dout(20) << "upkeep thread waiting interval " << interval << dendl;
-      upkeep_cvar.wait_for(lock, interval);
-    }
-  });
+  upkeeper = std::thread(&MDCache::upkeep_main, this);
 }
 
 MDCache::~MDCache() 
@@ -7754,7 +7714,6 @@ void MDCache::trim_client_leases()
   }
 }
 
-
 void MDCache::check_memory_usage()
 {
   static MemoryModel mm(g_ceph_context);
@@ -7780,10 +7739,6 @@ void MDCache::check_memory_usage()
   mds->update_mlogger();
   mds->mlogger->set(l_mdm_rss, last.get_rss());
   mds->mlogger->set(l_mdm_heap, last.get_heap());
-
-  if (cache_toofull()) {
-    mds->server->recall_client_state(nullptr, Server::RecallFlags::TRIM);
-  }
 }
 
 
@@ -13383,3 +13338,54 @@ void MDCache::handle_mdsmap(const MDSMap &mdsmap, const MDSMap &oldmap) {
     }
   }
 }
+
+void MDCache::upkeep_main(void)
+{
+  std::unique_lock lock(upkeep_mutex);
+  while (!upkeep_trim_shutdown.load()) {
+    auto now = clock::now();
+    auto since = now-upkeep_last_trim;
+    auto trim_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_trim_interval"));
+    if (since >= trim_interval*.90) {
+      lock.unlock(); /* mds_lock -> upkeep_mutex */
+      std::scoped_lock mds_lock(mds->mds_lock);
+      lock.lock();
+      if (upkeep_trim_shutdown.load())
+        return;
+      check_memory_usage();
+      if (mds->is_cache_trimmable()) {
+        dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl;
+        bool active_with_clients = mds->is_active() || mds->is_clientreplay() || mds->is_stopping();
+        if (active_with_clients) {
+          trim_client_leases();
+        }
+        trim();
+        if (active_with_clients) {
+          auto recall_flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS;
+          if (cache_toofull()) {
+            recall_flags = recall_flags|Server::RecallFlags::TRIM;
+          }
+          mds->server->recall_client_state(nullptr, recall_flags);
+        }
+        upkeep_last_trim = now = clock::now();
+      } else {
+        dout(10) << "cache not ready for trimming" << dendl;
+      }
+    } else {
+      trim_interval -= since;
+    }
+    since = now-upkeep_last_release;
+    auto release_interval = clock::duration(g_conf().get_val<std::chrono::seconds>("mds_cache_release_free_interval"));
+    if (since >= release_interval*.90) {
+      /* XXX not necessary once MDCache uses PriorityCache */
+      dout(10) << "releasing free memory" << dendl;
+      ceph_heap_release_free_memory();
+      upkeep_last_release = clock::now();
+    } else {
+      release_interval -= since;
+    }
+    auto interval = std::min(release_interval, trim_interval);
+    dout(20) << "upkeep thread waiting interval " << interval << dendl;
+    upkeep_cvar.wait_for(lock, interval);
+  }
+}

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 6d34aeee7537eae1361285948073f3095508a1c5..b2ef5e4e851fbb53ac67e3f3e3a67262e7d99faf 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1307,6 +1307,8 @@ class MDCache {
   void finish_uncommitted_fragment(dirfrag_t basedirfrag, int op);
   void rollback_uncommitted_fragment(dirfrag_t basedirfrag, frag_vec_t&& old_frags);
 
+  void upkeep_main(void);
+
   uint64_t cache_memory_limit;
   double cache_reservation;
   double cache_health_threshold;

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 7bcf7da77f75316ba134c100c1ab98a89b27ad00..bf259b35e9b09d676545533df783668e0f48473b 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -726,7 +726,7 @@ void MDSRankDispatcher::tick()
   }
 
   // ...
-  if (is_cache_trimmable()) {
+  if (is_clientreplay() || is_active() || is_stopping()) {
     server->find_idle_sessions();
     server->evict_cap_revoke_non_responders();
     locker->tick();

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index ee34c95d3158213567acfb6b6c5f5f7c461f8b3c..95741ee36ab948791a92fef4f635cbcafaccdd8c 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -194,7 +194,7 @@ class MDSRank {
     bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); }
 
     bool is_cache_trimmable() const {
-      return is_clientreplay() || is_active() || is_stopping();
+      return is_standby_replay() || is_clientreplay() || is_active() || is_stopping();
     }
 
     void handle_write_error(int err);