From: Patrick Donnelly Date: Tue, 30 Mar 2021 03:09:30 +0000 (-0700) Subject: mds: trim cache regularly for standby-replay X-Git-Tag: v17.1.0~2311^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=19293d9b9d19c32af4de655cd59e206056b2417d;p=ceph.git mds: trim cache regularly for standby-replay This change is slightly awkward because standby-replay MDS do not do all the kinds of upkeep a normal active MDS does. In particular, it is not going to recall client state from clients. This diff also merges the extra recall_client_state in MDCache::check_memory_usage into its only caller (the upkeep thread) where it was also doing a recall. That's just a matter of merging the recall flags. This has the added benefit of making MDCache::check_memory_usage callable for all MDS daemons regardless of state. Fixes: https://tracker.ceph.com/issues/50048 Signed-off-by: Patrick Donnelly --- diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index d48db912819e4..e3978e9edce3c 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -160,47 +160,7 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) : decayrate.set_halflife(g_conf()->mds_decay_halflife); - upkeeper = std::thread([this]() { - std::unique_lock lock(upkeep_mutex); - while (!upkeep_trim_shutdown.load()) { - auto now = clock::now(); - auto since = now-upkeep_last_trim; - auto trim_interval = clock::duration(g_conf().get_val("mds_cache_trim_interval")); - if (since >= trim_interval*.90) { - lock.unlock(); /* mds_lock -> upkeep_mutex */ - std::scoped_lock mds_lock(mds->mds_lock); - lock.lock(); - if (upkeep_trim_shutdown.load()) - return; - if (mds->is_cache_trimmable()) { - dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl; - trim_client_leases(); - trim(); - check_memory_usage(); - auto flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS; - mds->server->recall_client_state(nullptr, flags); - upkeep_last_trim = now = clock::now(); - } else { - dout(10) << "cache not ready for trimming" << dendl; - } - } else { - trim_interval -= since; - } - since = now-upkeep_last_release; - auto release_interval = clock::duration(g_conf().get_val("mds_cache_release_free_interval")); - if (since >= release_interval*.90) { - /* XXX not necessary once MDCache uses PriorityCache */ - dout(10) << "releasing free memory" << dendl; - ceph_heap_release_free_memory(); - upkeep_last_release = clock::now(); - } else { - release_interval -= since; - } - auto interval = std::min(release_interval, trim_interval); - dout(20) << "upkeep thread waiting interval " << interval << dendl; - upkeep_cvar.wait_for(lock, interval); - } - }); + upkeeper = std::thread(&MDCache::upkeep_main, this); } MDCache::~MDCache() @@ -7705,7 +7665,6 @@ void MDCache::trim_client_leases() } } - void MDCache::check_memory_usage() { static MemoryModel mm(g_ceph_context); @@ -7731,10 +7690,6 @@ void MDCache::check_memory_usage() mds->update_mlogger(); mds->mlogger->set(l_mdm_rss, last.get_rss()); mds->mlogger->set(l_mdm_heap, last.get_heap()); - - if (cache_toofull()) { - mds->server->recall_client_state(nullptr, Server::RecallFlags::TRIM); - } } @@ -13260,3 +13215,54 @@ void MDCache::handle_mdsmap(const MDSMap &mdsmap, const MDSMap &oldmap) { export_ephemeral_dist_frag_bits = n; } } + +void MDCache::upkeep_main(void) +{ + std::unique_lock lock(upkeep_mutex); + while (!upkeep_trim_shutdown.load()) { + auto now = clock::now(); + auto since = now-upkeep_last_trim; + auto trim_interval = clock::duration(g_conf().get_val("mds_cache_trim_interval")); + if (since >= trim_interval*.90) { + lock.unlock(); /* mds_lock -> upkeep_mutex */ + std::scoped_lock mds_lock(mds->mds_lock); + lock.lock(); + if (upkeep_trim_shutdown.load()) + return; + check_memory_usage(); + if (mds->is_cache_trimmable()) { + dout(20) << "upkeep thread trimming cache; last trim " << since << " ago" << dendl; + bool active_with_clients = mds->is_active() || mds->is_clientreplay() || mds->is_stopping(); + if (active_with_clients) { + trim_client_leases(); + } + trim(); + if (active_with_clients) { + auto recall_flags = Server::RecallFlags::ENFORCE_MAX|Server::RecallFlags::ENFORCE_LIVENESS; + if (cache_toofull()) { + recall_flags = recall_flags|Server::RecallFlags::TRIM; + } + mds->server->recall_client_state(nullptr, recall_flags); + } + upkeep_last_trim = now = clock::now(); + } else { + dout(10) << "cache not ready for trimming" << dendl; + } + } else { + trim_interval -= since; + } + since = now-upkeep_last_release; + auto release_interval = clock::duration(g_conf().get_val("mds_cache_release_free_interval")); + if (since >= release_interval*.90) { + /* XXX not necessary once MDCache uses PriorityCache */ + dout(10) << "releasing free memory" << dendl; + ceph_heap_release_free_memory(); + upkeep_last_release = clock::now(); + } else { + release_interval -= since; + } + auto interval = std::min(release_interval, trim_interval); + dout(20) << "upkeep thread waiting interval " << interval << dendl; + upkeep_cvar.wait_for(lock, interval); + } +} diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index d59d406751b96..1805c54e39602 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -1304,6 +1304,8 @@ class MDCache { void finish_uncommitted_fragment(dirfrag_t basedirfrag, int op); void rollback_uncommitted_fragment(dirfrag_t basedirfrag, frag_vec_t&& old_frags); + void upkeep_main(void); + uint64_t cache_memory_limit; double cache_reservation; double cache_health_threshold; diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 46c307ee41307..14607b9514a38 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -728,7 +728,7 @@ void MDSRankDispatcher::tick() } // ... - if (is_cache_trimmable()) { + if (is_clientreplay() || is_active() || is_stopping()) { server->find_idle_sessions(); server->evict_cap_revoke_non_responders(); locker->tick(); diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index 8d0027820f4ce..8e97c634da415 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -219,7 +219,7 @@ class MDSRank { bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); } bool is_cache_trimmable() const { - return is_clientreplay() || is_active() || is_stopping(); + return is_standby_replay() || is_clientreplay() || is_active() || is_stopping(); } void handle_write_error(int err);