From 20adfb08aadaef0c41b417f72ac1fd6dad0a6802 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 28 Aug 2024 16:34:38 +0800 Subject: [PATCH] crimson/os/seastore/cache: monitor cache access by extent and trans Signed-off-by: Yingxin Cheng --- .../seastore/backref/btree_backref_manager.cc | 2 + src/crimson/os/seastore/btree/fixed_kv_node.h | 2 + src/crimson/os/seastore/cache.cc | 38 ++++---- src/crimson/os/seastore/cache.h | 96 ++++++++++++++++++- .../lba_manager/btree/btree_lba_manager.cc | 2 + src/crimson/os/seastore/seastore_types.h | 32 +++++++ 6 files changed, 148 insertions(+), 24 deletions(-) diff --git a/src/crimson/os/seastore/backref/btree_backref_manager.cc b/src/crimson/os/seastore/backref/btree_backref_manager.cc index 30ff4554074ea..f89698d602aed 100644 --- a/src/crimson/os/seastore/backref/btree_backref_manager.cc +++ b/src/crimson/os/seastore/backref/btree_backref_manager.cc @@ -38,12 +38,14 @@ const get_phy_tree_root_node_ret get_phy_tree_root_node< trans_intr::make_interruptible( c.cache.get_extent_viewable_by_trans(c.trans, backref_root))}; } else { + c.cache.account_absent_access(c.trans.get_src()); return {false, trans_intr::make_interruptible( Cache::get_extent_ertr::make_ready_future< CachedExtentRef>())}; } } else { + c.cache.account_absent_access(c.trans.get_src()); return {false, trans_intr::make_interruptible( Cache::get_extent_ertr::make_ready_future< diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h index eacf8a8cc4012..09f54a4f2d0c3 100644 --- a/src/crimson/os/seastore/btree/fixed_kv_node.h +++ b/src/crimson/os/seastore/btree/fixed_kv_node.h @@ -370,9 +370,11 @@ struct FixedKVNode : ChildableCachedExtent { if (is_valid_child_ptr(child)) { return c.cache.template get_extent_viewable_by_trans(c.trans, (T*)child); } else { + c.cache.account_absent_access(c.trans.get_src()); return child_pos_t(&sparent, spos); } } else { + c.cache.account_absent_access(c.trans.get_src()); return child_pos_t(this, pos); } } diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 73e87d2bd9527..3ecff20feb3f4 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -204,25 +204,25 @@ void Cache::register_metrics() /* * cache_query: cache_access and cache_hit */ - for (auto& [src, src_label] : labels_by_src) { - metrics.add_group( - "cache", - { - sm::make_counter( - "cache_access", - get_by_src(stats.cache_query_by_src, src).access, - sm::description("total number of cache accesses"), - {src_label} - ), - sm::make_counter( - "cache_hit", - get_by_src(stats.cache_query_by_src, src).hit, - sm::description("total number of cache hits"), - {src_label} - ), - } - ); - } + metrics.add_group( + "cache", + { + sm::make_counter( + "cache_access", + [this] { + return stats.access.get_cache_access(); + }, + sm::description("total number of cache accesses") + ), + sm::make_counter( + "cache_hit", + [this] { + return stats.access.s.get_cache_hit(); + }, + sm::description("total number of cache hits") + ), + } + ); { /* diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 647e146a3a566..b2ab0e8c0cb47 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -272,6 +272,11 @@ public: return t.root; } + void account_absent_access(Transaction::src_t src) { + ++(get_by_src(stats.cache_absent_by_src, src)); + ++stats.access.cache_absent; + } + /** * get_extent_if_cached * @@ -287,12 +292,29 @@ public: CachedExtentRef ret; LOG_PREFIX(Cache::get_extent_if_cached); auto result = t.get_extent(offset, &ret); + const auto t_src = t.get_src(); + extent_access_stats_t& access_stats = get_by_ext( + get_by_src(stats.access_by_src_ext, t_src), + type); if (result == Transaction::get_extent_ret::RETIRED) { SUBDEBUGT(seastore_cache, "{} {} is retired on t -- {}", t, type, offset, *ret); return get_extent_if_cached_iertr::make_ready_future< CachedExtentRef>(ret); } else if (result == Transaction::get_extent_ret::PRESENT) { + if (ret->is_stable()) { + if (ret->is_dirty()) { + ++access_stats.trans_dirty; + ++stats.access.s.trans_dirty; + } else { + ++access_stats.trans_lru; + ++stats.access.s.trans_lru; + } + } else { + ++access_stats.trans_pending; + ++stats.access.s.trans_pending; + } + if (ret->is_fully_loaded()) { SUBTRACET(seastore_cache, "{} {} is present on t -- {}", t, type, offset, *ret); @@ -313,13 +335,26 @@ public: ret = query_cache(offset, &metric_key); if (!ret) { SUBDEBUGT(seastore_cache, "{} {} is absent", t, type, offset); + account_absent_access(t_src); return get_extent_if_cached_iertr::make_ready_future(); } else if (is_retired_placeholder_type(ret->get_type())) { // retired_placeholder is not really cached yet SUBDEBUGT(seastore_cache, "{} {} is absent(placeholder)", t, type, offset); + account_absent_access(t_src); return get_extent_if_cached_iertr::make_ready_future(); - } else if (!ret->is_fully_loaded()) { + } + + if (ret->is_dirty()) { + ++access_stats.cache_dirty; + ++stats.access.s.cache_dirty; + } else { + ++access_stats.cache_lru; + ++stats.access.s.cache_lru; + } + + if (!ret->is_fully_loaded()) { + // ignore non-full extent SUBDEBUGT(seastore_cache, "{} {} is present without " "being fully loaded", t, type, offset); return get_extent_if_cached_iertr::make_ready_future(); @@ -329,7 +364,6 @@ public: SUBDEBUGT(seastore_cache, "{} {} is present in cache -- {}", t, type, offset, *ret); t.add_to_read_set(ret); - const auto t_src = t.get_src(); touch_extent(*ret, &t_src); return ret->wait_io().then([ret] { return get_extent_if_cached_iertr::make_ready_future< @@ -349,6 +383,8 @@ public: * * Note, the current implementation leverages parent-child * pointers in LBA instead, so it should only be called in tests. + * + * This path won't be accounted by the cache_access_stats_t. */ using get_extent_iertr = base_iertr; template @@ -426,9 +462,14 @@ public: // FIXME: assert(ext.is_stable_clean()); assert(ext.is_stable()); assert(T::TYPE == ext.get_type()); + const auto t_src = t.get_src(); + extent_access_stats_t& access_stats = get_by_ext( + get_by_src(stats.access_by_src_ext, t_src), + T::TYPE); + ++access_stats.load_absent; + ++stats.access.s.load_absent; t.add_to_read_set(CachedExtentRef(&ext)); - const auto t_src = t.get_src(); touch_extent(ext, &t_src); }; auto metric_key = std::make_pair(t.get_src(), T::TYPE); @@ -484,6 +525,13 @@ public: CachedExtentRef extent) { assert(extent->is_valid()); + + const auto t_src = t.get_src(); + auto ext_type = extent->get_type(); + extent_access_stats_t& access_stats = get_by_ext( + get_by_src(stats.access_by_src_ext, t_src), + ext_type); + CachedExtent* p_extent; if (extent->is_stable()) { p_extent = extent->get_transactional_view(t); @@ -491,6 +539,8 @@ public: assert(!extent->is_stable_writting()); assert(p_extent->is_pending_in_trans(t.get_trans_id())); assert(!p_extent->is_stable_writting()); + ++access_stats.trans_pending; + ++stats.access.s.trans_pending; if (p_extent->is_mutable()) { assert(p_extent->is_fully_loaded()); assert(!p_extent->is_pending_io()); @@ -503,13 +553,29 @@ public: // stable from trans-view assert(!p_extent->is_pending_in_trans(t.get_trans_id())); if (t.maybe_add_to_read_set(p_extent)) { - const auto t_src = t.get_src(); + if (p_extent->is_dirty()) { + ++access_stats.cache_dirty; + ++stats.access.s.cache_dirty; + } else { + ++access_stats.cache_lru; + ++stats.access.s.cache_lru; + } touch_extent(*p_extent, &t_src); + } else { + if (p_extent->is_dirty()) { + ++access_stats.trans_dirty; + ++stats.access.s.trans_dirty; + } else { + ++access_stats.trans_lru; + ++stats.access.s.trans_lru; + } } } } else { assert(!extent->is_stable_writting()); assert(extent->is_pending_in_trans(t.get_trans_id())); + ++access_stats.trans_pending; + ++stats.access.s.trans_pending; if (extent->is_mutable()) { assert(extent->is_fully_loaded()); assert(!extent->is_pending_io()); @@ -524,6 +590,8 @@ public: ceph_assert(!is_retired_placeholder_type(p_extent->get_type())); if (!p_extent->is_fully_loaded()) { assert(!p_extent->is_mutable()); + ++access_stats.load_present; + ++stats.access.s.load_present; LOG_PREFIX(Cache::get_extent_viewable_by_trans); SUBDEBUG(seastore_cache, "{} {}~{} is present without been fully loaded, reading ... -- {}", @@ -693,6 +761,14 @@ private: extent_init_func_t &&on_cache ); + /** + * get_caching_extent_by_type + * + * Note, the current implementation leverages parent-child + * pointers in LBA instead, so it should only be called in tests. + * + * This path won't be accounted by the cache_access_stats_t. + */ using get_extent_by_type_iertr = get_extent_iertr; using get_extent_by_type_ret = get_extent_by_type_iertr::future< CachedExtentRef>; @@ -768,9 +844,14 @@ private: auto f = [&t, this](CachedExtent &ext) { // FIXME: assert(ext.is_stable_clean()); assert(ext.is_stable()); + const auto t_src = t.get_src(); + extent_access_stats_t& access_stats = get_by_ext( + get_by_src(stats.access_by_src_ext, t_src), + ext.get_type()); + ++access_stats.load_absent; + ++stats.access.s.load_absent; t.add_to_read_set(CachedExtentRef(&ext)); - const auto t_src = t.get_src(); touch_extent(ext, &t_src); }; auto src = t.get_src(); @@ -1614,6 +1695,11 @@ private: counter_by_src_t > dirty_io_by_src_ext; + cache_access_stats_t access; + counter_by_src_t cache_absent_by_src; + counter_by_src_t > + access_by_src_ext; + uint64_t onode_tree_depth = 0; int64_t onode_tree_extents_num = 0; counter_by_src_t committed_onode_tree_efforts; diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc index 8439b733704e4..b7a1d8f8ba966 100644 --- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc @@ -62,12 +62,14 @@ const get_phy_tree_root_node_ret get_phy_tree_root_node< trans_intr::make_interruptible( c.cache.get_extent_viewable_by_trans(c.trans, lba_root))}; } else { + c.cache.account_absent_access(c.trans.get_src()); return {false, trans_intr::make_interruptible( Cache::get_extent_ertr::make_ready_future< CachedExtentRef>())}; } } else { + c.cache.account_absent_access(c.trans.get_src()); return {false, trans_intr::make_interruptible( Cache::get_extent_ertr::make_ready_future< diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 5cdb1f8614239..ba2928848deb0 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -2868,6 +2868,38 @@ struct dirty_io_stats_printer_t { }; std::ostream& operator<<(std::ostream&, const dirty_io_stats_printer_t&); +/* + * Doesn't account: + * replay + * rewrite + * retiring/placeholder + * get_caching_extent() -- test only + * get_caching_extent_by_type() -- test only + */ +struct extent_access_stats_t { + uint64_t trans_pending = 0; + uint64_t trans_dirty = 0; + uint64_t trans_lru = 0; + uint64_t cache_dirty = 0; + uint64_t cache_lru = 0; + + uint64_t load_absent = 0; + uint64_t load_present = 0; + + uint64_t get_cache_hit() const { + return cache_dirty + cache_lru; + } +}; + +struct cache_access_stats_t { + extent_access_stats_t s; + uint64_t cache_absent = 0; + + uint64_t get_cache_access() const { + return s.get_cache_hit() + cache_absent; + } +}; + struct cache_stats_t { cache_size_stats_t lru_sizes; cache_io_stats_t lru_io; -- 2.39.5