From ca9d3b618db27d55266f266eab8a0aeacbad9fce Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 29 Mar 2021 20:09:30 -0700 Subject: [PATCH] mds: trim cache regularly for standby-replay This change is slightly awkward because standby-replay MDS do not do all the kinds of upkeep a normal active MDS does. In particular, it is not going to recall client state from clients. This diff also merges the extra recall_client_state in MDCache::check_memory_usage into its only caller (the upkeep thread) where it was also doing a recall. That's just a matter of merging the recall flags. This has the added benefit of making MDCache::check_memory_usage callable for all MDS daemons regardless of state. Fixes: https://tracker.ceph.com/issues/50048 Signed-off-by: Patrick Donnelly (cherry picked from commit 19293d9b9d19c32af4de655cd59e206056b2417d) --- src/mds/MDCache.cc | 98 ++++++++++++++++++++++++---------------------- src/mds/MDCache.h | 2 + src/mds/MDSRank.cc | 2 +- src/mds/MDSRank.h | 2 +- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 842ad7d860190..8236b429dc488 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -162,47 +162,7 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) : decayrate.set_halflife(g_conf()->mds_decay_halflife); - upkeeper = std::thread([this]() { - std::unique_lock lock(upkeep_mutex); - while (!upkeep_trim_shutdown.load()) { - auto now = clock::now(); - auto since = now-upkeep_last_trim; - auto trim_interval = clock::duration(g_conf().get_val("mds_cache_trim_interval")); - if (since >= trim_interval*.90) { - lock.unlock(); /* mds_lock -> upkeep_mutex */ - std::scoped_lock mds_lock(mds->mds_lock); - lock.lock(); - if (upkeep_trim_shutdown.load()) - return; - if (mds->is_cache_trimmable()) { - dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl; - trim_client_leases(); - trim(); - check_memory_usage(); - auto flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS; - mds->server->recall_client_state(nullptr, flags); - upkeep_last_trim = now = clock::now(); - } else { - dout(10) << "cache not ready for trimming" << dendl; - } - } else { - trim_interval -= since; - } - since = now-upkeep_last_release; - auto release_interval = clock::duration(g_conf().get_val("mds_cache_release_free_interval")); - if (since >= release_interval*.90) { - /* XXX not necessary once MDCache uses PriorityCache */ - dout(10) << "releasing free memory" << dendl; - ceph_heap_release_free_memory(); - upkeep_last_release = clock::now(); - } else { - release_interval -= since; - } - auto interval = std::min(release_interval, trim_interval); - dout(20) << "upkeep thread waiting interval " << interval << dendl; - upkeep_cvar.wait_for(lock, interval); - } - }); + upkeeper = std::thread(&MDCache::upkeep_main, this); } MDCache::~MDCache() @@ -7754,7 +7714,6 @@ void MDCache::trim_client_leases() } } - void MDCache::check_memory_usage() { static MemoryModel mm(g_ceph_context); @@ -7780,10 +7739,6 @@ void MDCache::check_memory_usage() mds->update_mlogger(); mds->mlogger->set(l_mdm_rss, last.get_rss()); mds->mlogger->set(l_mdm_heap, last.get_heap()); - - if (cache_toofull()) { - mds->server->recall_client_state(nullptr, Server::RecallFlags::TRIM); - } } @@ -13383,3 +13338,54 @@ void MDCache::handle_mdsmap(const MDSMap &mdsmap, const MDSMap &oldmap) { } } } + +void MDCache::upkeep_main(void) +{ + std::unique_lock lock(upkeep_mutex); + while (!upkeep_trim_shutdown.load()) { + auto now = clock::now(); + auto since = now-upkeep_last_trim; + auto trim_interval = clock::duration(g_conf().get_val("mds_cache_trim_interval")); + if (since >= trim_interval*.90) { + lock.unlock(); /* mds_lock -> upkeep_mutex */ + std::scoped_lock mds_lock(mds->mds_lock); + lock.lock(); + if (upkeep_trim_shutdown.load()) + return; + check_memory_usage(); + if (mds->is_cache_trimmable()) { + dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl; + bool active_with_clients = mds->is_active() || mds->is_clientreplay() || mds->is_stopping(); + if (active_with_clients) { + trim_client_leases(); + } + trim(); + if (active_with_clients) { + auto recall_flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS; + if (cache_toofull()) { + recall_flags = recall_flags|Server::RecallFlags::TRIM; + } + mds->server->recall_client_state(nullptr, recall_flags); + } + upkeep_last_trim = now = clock::now(); + } else { + dout(10) << "cache not ready for trimming" << dendl; + } + } else { + trim_interval -= since; + } + since = now-upkeep_last_release; + auto release_interval = clock::duration(g_conf().get_val("mds_cache_release_free_interval")); + if (since >= release_interval*.90) { + /* XXX not necessary once MDCache uses PriorityCache */ + dout(10) << "releasing free memory" << dendl; + ceph_heap_release_free_memory(); + upkeep_last_release = clock::now(); + } else { + release_interval -= since; + } + auto interval = std::min(release_interval, trim_interval); + dout(20) << "upkeep thread waiting interval " << interval << dendl; + upkeep_cvar.wait_for(lock, interval); + } +} diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index 6d34aeee7537e..b2ef5e4e851fb 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -1307,6 +1307,8 @@ class MDCache { void finish_uncommitted_fragment(dirfrag_t basedirfrag, int op); void rollback_uncommitted_fragment(dirfrag_t basedirfrag, frag_vec_t&& old_frags); + void upkeep_main(void); + uint64_t cache_memory_limit; double cache_reservation; double cache_health_threshold; diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 7bcf7da77f753..bf259b35e9b09 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -726,7 +726,7 @@ void MDSRankDispatcher::tick() } // ... - if (is_cache_trimmable()) { + if (is_clientreplay() || is_active() || is_stopping()) { server->find_idle_sessions(); server->evict_cap_revoke_non_responders(); locker->tick(); diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index ee34c95d31582..95741ee36ab94 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -194,7 +194,7 @@ class MDSRank { bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); } bool is_cache_trimmable() const { - return is_clientreplay() || is_active() || is_stopping(); + return is_standby_replay() || is_clientreplay() || is_active() || is_stopping(); } void handle_write_error(int err); -- 2.39.5