From 722847e14ec4206baab5bc80510562e47b71a4a8 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 29 Mar 2021 20:09:30 -0700 Subject: [PATCH] mds: trim cache regularly for standby-replay This change is slightly awkward because standby-replay MDS do not do all the kinds of upkeep a normal active MDS does. In particular, it is not going to recall client state from clients. This diff also merges the extra recall_client_state in MDCache::check_memory_usage into its only caller (the upkeep thread) where it was also doing a recall. That's just a matter of merging the recall flags. This has the added benefit of making MDCache::check_memory_usage callable for all MDS daemons regardless of state. Fixes: https://tracker.ceph.com/issues/50048 Signed-off-by: Patrick Donnelly (cherry picked from commit 19293d9b9d19c32af4de655cd59e206056b2417d) Conflicts: src/mds/MDCache.cc - conflict was caused by an additional line "upkeep_last_trim = clock::now();" which should have been dropped by da45b4e7ca3a1c88c44424256c104b07710e9679 (nautilus cherry-pick of master commit af4cac5ec7bab4e5bf8936cd685f6ee8dcb38127) src/mds/MDCache.h --- src/mds/MDCache.cc | 98 ++++++++++++++++++++++++---------------------- src/mds/MDCache.h | 1 + src/mds/MDSRank.cc | 2 +- src/mds/MDSRank.h | 2 +- 4 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 837c5f4dc2527..83497ff06dfc2 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -159,48 +159,7 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) : decayrate.set_halflife(g_conf()->mds_decay_halflife); - upkeeper = std::thread([this]() { - std::unique_lock lock(upkeep_mutex); - while (!upkeep_trim_shutdown.load()) { - auto now = clock::now(); - auto since = now-upkeep_last_trim; - auto trim_interval = clock::duration(g_conf().get_val("mds_cache_trim_interval")); - if (since >= trim_interval*.90) { - lock.unlock(); /* mds_lock -> upkeep_mutex */ - std::scoped_lock mds_lock(mds->mds_lock); - lock.lock(); - if (upkeep_trim_shutdown.load()) - return; - if (mds->is_cache_trimmable()) { - dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl; - trim_client_leases(); - trim(); - check_memory_usage(); - auto flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS; - mds->server->recall_client_state(nullptr, flags); - upkeep_last_trim = clock::now(); - upkeep_last_trim = now = clock::now(); - } else { - dout(10) << "cache not ready for trimming" << dendl; - } - } else { - trim_interval -= since; - } - since = now-upkeep_last_release; - auto release_interval = clock::duration(g_conf().get_val("mds_cache_release_free_interval")); - if (since >= release_interval*.90) { - /* XXX not necessary once MDCache uses PriorityCache */ - dout(10) << "releasing free memory" << dendl; - ceph_heap_release_free_memory(); - upkeep_last_release = clock::now(); - } else { - release_interval -= since; - } - auto interval = std::min(release_interval, trim_interval); - dout(20) << "upkeep thread waiting interval " << interval << dendl; - upkeep_cvar.wait_for(lock, interval); - } - }); + upkeeper = std::thread(&MDCache::upkeep_main, this); } MDCache::~MDCache() @@ -7632,7 +7591,6 @@ void MDCache::trim_client_leases() } } - void MDCache::check_memory_usage() { static MemoryModel mm(g_ceph_context); @@ -7658,10 +7616,6 @@ void MDCache::check_memory_usage() mds->update_mlogger(); mds->mlogger->set(l_mdm_rss, last.get_rss()); mds->mlogger->set(l_mdm_heap, last.get_heap()); - - if (cache_toofull()) { - mds->server->recall_client_state(nullptr, Server::RecallFlags::TRIM); - } } @@ -13064,3 +13018,53 @@ void MDCache::handle_mdsmap(const MDSMap &mdsmap) { } } +void MDCache::upkeep_main(void) +{ + std::unique_lock lock(upkeep_mutex); + while (!upkeep_trim_shutdown.load()) { + auto now = clock::now(); + auto since = now-upkeep_last_trim; + auto trim_interval = clock::duration(g_conf().get_val("mds_cache_trim_interval")); + if (since >= trim_interval*.90) { + lock.unlock(); /* mds_lock -> upkeep_mutex */ + std::scoped_lock mds_lock(mds->mds_lock); + lock.lock(); + if (upkeep_trim_shutdown.load()) + return; + check_memory_usage(); + if (mds->is_cache_trimmable()) { + dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl; + bool active_with_clients = mds->is_active() || mds->is_clientreplay() || mds->is_stopping(); + if (active_with_clients) { + trim_client_leases(); + } + trim(); + if (active_with_clients) { + auto recall_flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS; + if (cache_toofull()) { + recall_flags = recall_flags|Server::RecallFlags::TRIM; + } + mds->server->recall_client_state(nullptr, recall_flags); + } + upkeep_last_trim = now = clock::now(); + } else { + dout(10) << "cache not ready for trimming" << dendl; + } + } else { + trim_interval -= since; + } + since = now-upkeep_last_release; + auto release_interval = clock::duration(g_conf().get_val("mds_cache_release_free_interval")); + if (since >= release_interval*.90) { + /* XXX not necessary once MDCache uses PriorityCache */ + dout(10) << "releasing free memory" << dendl; + ceph_heap_release_free_memory(); + upkeep_last_release = clock::now(); + } else { + release_interval -= since; + } + auto interval = std::min(release_interval, trim_interval); + dout(20) << "upkeep thread waiting interval " << interval << dendl; + upkeep_cvar.wait_for(lock, interval); + } +} diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index 10946be8fca58..655ea8659dbbb 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -157,6 +157,7 @@ class MDCache { Filer filer; private: + void upkeep_main(void); uint64_t cache_inode_limit; uint64_t cache_memory_limit; double cache_reservation; diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 83a9d1273c940..6e1f79833dbe5 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -746,7 +746,7 @@ void MDSRankDispatcher::tick() } // ... - if (is_cache_trimmable()) { + if (is_clientreplay() || is_active() || is_stopping()) { server->find_idle_sessions(); server->evict_cap_revoke_non_responders(); locker->tick(); diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index c10bad2307b06..d07b2ffddb797 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -234,7 +234,7 @@ class MDSRank { bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); } bool is_cache_trimmable() const { - return is_clientreplay() || is_active() || is_stopping(); + return is_standby_replay() || is_clientreplay() || is_active() || is_stopping(); } void handle_write_error(int err); -- 2.39.5