From: Chunmei Liu Date: Wed, 28 Jan 2026 00:11:07 +0000 (+0000) Subject: crimson/os/seastore: make register_metrics works for X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=21cc1e1ec4634db8466aabd1bcb613ef14a32666;p=ceph-ci.git crimson/os/seastore: make register_metrics works for multiple store shards on one reactor Signed-off-by: Chunmei Liu --- diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc index c52bcb75659..52515354769 100644 --- a/src/crimson/os/seastore/async_cleaner.cc +++ b/src/crimson/os/seastore/async_cleaner.cc @@ -410,6 +410,7 @@ JournalTrimmerImpl::config_t::get_test( } JournalTrimmerImpl::JournalTrimmerImpl( + unsigned int store_index, BackrefManager &backref_manager, config_t config, backend_type_t type, @@ -427,7 +428,7 @@ JournalTrimmerImpl::JournalTrimmerImpl( config.validate(); ceph_assert(roll_start >= 0); ceph_assert(roll_size > 0); - register_metrics(); + register_metrics(store_index); } void JournalTrimmerImpl::set_journal_head(journal_seq_t head) @@ -734,16 +735,20 @@ JournalTrimmerImpl::trim_dirty() }); } -void JournalTrimmerImpl::register_metrics() +void JournalTrimmerImpl::register_metrics(unsigned int store_index) { namespace sm = seastar::metrics; metrics.add_group("journal_trimmer", { sm::make_counter("dirty_journal_bytes", [this] { return get_dirty_journal_size(); }, - sm::description("the size of the journal for dirty extents")), + sm::description("the size of the journal for dirty extents"), + {sm::label_instance("shard_store_index", + std::to_string(store_index))}), sm::make_counter("alloc_journal_bytes", [this] { return get_alloc_journal_size(); }, - sm::description("the size of the journal for alloc info")) + sm::description("the size of the journal for alloc info"), + {sm::label_instance("shard_store_index", + std::to_string(store_index))}), }); } @@ -907,6 +912,7 @@ std::ostream &operator<<( } SegmentCleaner::SegmentCleaner( + unsigned int store_index, config_t config, SegmentManagerGroupRef&& sm_group, BackrefManager &backref_manager, @@ -914,7 +920,8 @@ SegmentCleaner::SegmentCleaner( rewrite_gen_t max_rewrite_generation, bool detailed, bool is_cold) - : detailed(detailed), + : store_index(store_index), + detailed(detailed), is_cold(is_cold), config(config), sm_group(std::move(sm_group)), @@ -960,96 +967,127 @@ void SegmentCleaner::register_metrics() metrics.add_group(prefix, { sm::make_counter("segments_number", [this] { return segments.get_num_segments(); }, - sm::description("the number of segments")), + sm::description("the number of segments"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segment_size", [this] { return segments.get_segment_size(); }, - sm::description("the bytes of a segment")), + sm::description("the bytes of a segment"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_in_journal", [this] { return get_segments_in_journal(); }, - sm::description("the number of segments in journal")), + sm::description("the number of segments in journal"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_type_journal", [this] { return segments.get_num_type_journal(); }, - sm::description("the number of segments typed journal")), + sm::description("the number of segments typed journal"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_type_ool", [this] { return segments.get_num_type_ool(); }, - sm::description("the number of segments typed out-of-line")), + sm::description("the number of segments typed out-of-line"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_open", [this] { return segments.get_num_open(); }, - sm::description("the number of open segments")), + sm::description("the number of open segments"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_empty", [this] { return segments.get_num_empty(); }, - sm::description("the number of empty segments")), + sm::description("the number of empty segments"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_closed", [this] { return segments.get_num_closed(); }, - sm::description("the number of closed segments")), + sm::description("the number of closed segments"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_count_open_journal", [this] { return segments.get_count_open_journal(); }, - sm::description("the count of open journal segment operations")), + sm::description("the count of open journal segment operations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_count_open_ool", [this] { return segments.get_count_open_ool(); }, - sm::description("the count of open ool segment operations")), + sm::description("the count of open ool segment operations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_count_release_journal", [this] { return segments.get_count_release_journal(); }, - sm::description("the count of release journal segment operations")), + sm::description("the count of release journal segment operations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_count_release_ool", [this] { return segments.get_count_release_ool(); }, - sm::description("the count of release ool segment operations")), + sm::description("the count of release ool segment operations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_count_close_journal", [this] { return segments.get_count_close_journal(); }, - sm::description("the count of close journal segment operations")), + sm::description("the count of close journal segment operations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("segments_count_close_ool", [this] { return segments.get_count_close_ool(); }, - sm::description("the count of close ool segment operations")), + sm::description("the count of close ool segment operations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("total_bytes", [this] { return segments.get_total_bytes(); }, - sm::description("the size of the space")), + sm::description("the size of the space"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("available_bytes", [this] { return segments.get_available_bytes(); }, - sm::description("the size of the space is available")), + sm::description("the size of the space is available"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("unavailable_unreclaimable_bytes", [this] { return get_unavailable_unreclaimable_bytes(); }, - sm::description("the size of the space is unavailable and unreclaimable")), + sm::description("the size of the space is unavailable and unreclaimable"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("unavailable_reclaimable_bytes", [this] { return get_unavailable_reclaimable_bytes(); }, - sm::description("the size of the space is unavailable and reclaimable")), + sm::description("the size of the space is unavailable and reclaimable"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("used_bytes", stats.used_bytes, - sm::description("the size of the space occupied by live extents")), + sm::description("the size of the space occupied by live extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("unavailable_unused_bytes", [this] { return get_unavailable_unused_bytes(); }, - sm::description("the size of the space is unavailable and not alive")), + sm::description("the size of the space is unavailable and not alive"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("projected_count", stats.projected_count, - sm::description("the number of projected usage reservations")), + sm::description("the number of projected usage reservations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("projected_used_bytes_sum", stats.projected_used_bytes_sum, - sm::description("the sum of the projected usage in bytes")), + sm::description("the sum of the projected usage in bytes"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("reclaimed_bytes", stats.reclaimed_bytes, - sm::description("rewritten bytes due to reclaim")), + sm::description("rewritten bytes due to reclaim"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("reclaimed_segment_bytes", stats.reclaimed_segment_bytes, - sm::description("rewritten bytes due to reclaim")), + sm::description("rewritten bytes due to reclaim"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("closed_journal_used_bytes", stats.closed_journal_used_bytes, - sm::description("used bytes when close a journal segment")), + sm::description("used bytes when close a journal segment"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("closed_journal_total_bytes", stats.closed_journal_total_bytes, - sm::description("total bytes of closed journal segments")), + sm::description("total bytes of closed journal segments"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("closed_ool_used_bytes", stats.closed_ool_used_bytes, - sm::description("used bytes when close a ool segment")), + sm::description("used bytes when close a ool segment"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("closed_ool_total_bytes", stats.closed_ool_total_bytes, - sm::description("total bytes of closed ool segments")), + sm::description("total bytes of closed ool segments"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_gauge("available_ratio", [this] { return segments.get_available_ratio(); }, - sm::description("ratio of available space to total space")), + sm::description("ratio of available space to total space"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_gauge("reclaim_ratio", [this] { return get_reclaim_ratio(); }, - sm::description("ratio of reclaimable space to unavailable space")), + sm::description("ratio of reclaimable space to unavailable space"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_histogram("segment_utilization_distribution", [this]() -> seastar::metrics::histogram& { return stats.segment_util; }, - sm::description("utilization distribution of all segments")) + sm::description("utilization distribution of all segments"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}) }); } @@ -1769,11 +1807,13 @@ void SegmentCleaner::print(std::ostream &os, bool is_detailed) const } RBMCleaner::RBMCleaner( + unsigned int store_index, RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, LBAManager &lba_manager, bool detailed) - : detailed(detailed), + : store_index(store_index), + detailed(detailed), rb_group(std::move(rb_group)), backref_manager(backref_manager), lba_manager(lba_manager) @@ -1985,12 +2025,15 @@ void RBMCleaner::register_metrics() metrics.add_group("rbm_cleaner", { sm::make_counter("total_bytes", [this] { return get_total_bytes(); }, - sm::description("the size of the space")), + sm::description("the size of the space"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("available_bytes", [this] { return get_total_bytes() - get_journal_bytes() - stats.used_bytes; }, - sm::description("the size of the space is available")), + sm::description("the size of the space is available"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("used_bytes", stats.used_bytes, - sm::description("the size of the space occupied by live extents")), + sm::description("the size of the space occupied by live extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}) }); } diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 0f335f4edfb..f544b53a324 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -537,6 +537,7 @@ public: }; JournalTrimmerImpl( + unsigned int store_index, BackrefManager &backref_manager, config_t config, backend_type_t type, @@ -625,6 +626,7 @@ public: seastar::future<> trim(); static JournalTrimmerImplRef create( + unsigned int store_index, BackrefManager &backref_manager, config_t config, backend_type_t type, @@ -632,6 +634,7 @@ public: device_off_t roll_size, bool tail_include_alloc) { return std::make_unique( + store_index, backref_manager, config, type, roll_start, roll_size, tail_include_alloc); } @@ -693,7 +696,7 @@ private: return std::min(get_max_dirty_bytes_to_trim(), config.rewrite_dirty_bytes_per_cycle); } - void register_metrics(); + void register_metrics(unsigned int store_index); ExtentCallbackInterface *extent_callback = nullptr; BackgroundListener *background_callback = nullptr; @@ -1304,6 +1307,7 @@ public: }; SegmentCleaner( + unsigned int store_index, config_t config, SegmentManagerGroupRef&& sm_group, BackrefManager &backref_manager, @@ -1317,6 +1321,7 @@ public: } static SegmentCleanerRef create( + unsigned int store_index, config_t config, SegmentManagerGroupRef&& sm_group, BackrefManager &backref_manager, @@ -1324,7 +1329,7 @@ public: rewrite_gen_t max_rewrite_generation, bool detailed, bool is_cold = false) { - return std::make_unique( + return std::make_unique(store_index, config, std::move(sm_group), backref_manager, ool_seq_allocator, max_rewrite_generation, detailed, is_cold); @@ -1643,6 +1648,7 @@ private: } } + unsigned int store_index; const bool detailed; const bool is_cold; const config_t config; @@ -1710,17 +1716,20 @@ using RBMCleanerRef = std::unique_ptr; class RBMCleaner : public AsyncCleaner { public: RBMCleaner( + unsigned int store_index, RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, LBAManager &lba_manager, bool detailed); static RBMCleanerRef create( + unsigned int store_index, RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, LBAManager &lba_manager, bool detailed) { return std::make_unique( + store_index, std::move(rb_group), backref_manager, lba_manager, detailed); } @@ -1861,6 +1870,7 @@ public: private: bool equals(const RBMSpaceTracker &other) const; + unsigned int store_index; const bool detailed; RBMDeviceGroupRef rb_group; BackrefManager &backref_manager; diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 864654e3488..55f581da8c5 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -29,13 +29,14 @@ SET_SUBSYS(seastore_cache); namespace crimson::os::seastore { Cache::Cache( - ExtentPlacementManager &epm) + ExtentPlacementManager &epm, + unsigned int store_index) : epm(epm), pinboard(create_extent_pinboard( crimson::common::get_conf( "seastore_cachepin_size_pershard"))) { - register_metrics(); + register_metrics(store_index); segment_providers_by_device_id.resize(DEVICE_ID_MAX, nullptr); } @@ -130,7 +131,7 @@ void Cache::dump_contents() DEBUG("exit"); } -void Cache::register_metrics() +void Cache::register_metrics(unsigned int store_index) { LOG_PREFIX(Cache::register_metrics); DEBUG(""); @@ -147,37 +148,42 @@ void Cache::register_metrics() namespace sm = seastar::metrics; using src_t = Transaction::src_t; - std::map labels_by_src { - {src_t::MUTATE, sm::label_instance("src", "MUTATE")}, - {src_t::READ, sm::label_instance("src", "READ")}, - {src_t::TRIM_DIRTY, sm::label_instance("src", "TRIM_DIRTY")}, - {src_t::TRIM_ALLOC, sm::label_instance("src", "TRIM_ALLOC")}, - {src_t::CLEANER_MAIN, sm::label_instance("src", "CLEANER_MAIN")}, - {src_t::CLEANER_COLD, sm::label_instance("src", "CLEANER_COLD")}, + std::map> labels_by_src { + {src_t::MUTATE, {sm::label_instance("src", "MUTATE")}}, + {src_t::READ, {sm::label_instance("src", "READ")}}, + {src_t::TRIM_DIRTY, {sm::label_instance("src", "TRIM_DIRTY")}}, + {src_t::TRIM_ALLOC, {sm::label_instance("src", "TRIM_ALLOC")}}, + {src_t::CLEANER_MAIN, {sm::label_instance("src", "CLEANER_MAIN")}}, + {src_t::CLEANER_COLD, {sm::label_instance("src", "CLEANER_COLD")}}, }; assert(labels_by_src.size() == (std::size_t)src_t::MAX); - std::map labels_by_ext { - {extent_types_t::ROOT, sm::label_instance("ext", "ROOT")}, - {extent_types_t::LADDR_INTERNAL, sm::label_instance("ext", "LADDR_INTERNAL")}, - {extent_types_t::LADDR_LEAF, sm::label_instance("ext", "LADDR_LEAF")}, - {extent_types_t::DINK_LADDR_LEAF, sm::label_instance("ext", "DINK_LADDR_LEAF")}, - {extent_types_t::ROOT_META, sm::label_instance("ext", "ROOT_META")}, - {extent_types_t::OMAP_INNER, sm::label_instance("ext", "OMAP_INNER")}, - {extent_types_t::OMAP_LEAF, sm::label_instance("ext", "OMAP_LEAF")}, - {extent_types_t::ONODE_BLOCK_STAGED, sm::label_instance("ext", "ONODE_BLOCK_STAGED")}, - {extent_types_t::COLL_BLOCK, sm::label_instance("ext", "COLL_BLOCK")}, - {extent_types_t::OBJECT_DATA_BLOCK, sm::label_instance("ext", "OBJECT_DATA_BLOCK")}, - {extent_types_t::RETIRED_PLACEHOLDER, sm::label_instance("ext", "RETIRED_PLACEHOLDER")}, - {extent_types_t::ALLOC_INFO, sm::label_instance("ext", "ALLOC_INFO")}, - {extent_types_t::JOURNAL_TAIL, sm::label_instance("ext", "JOURNAL_TAIL")}, - {extent_types_t::TEST_BLOCK, sm::label_instance("ext", "TEST_BLOCK")}, - {extent_types_t::TEST_BLOCK_PHYSICAL, sm::label_instance("ext", "TEST_BLOCK_PHYSICAL")}, - {extent_types_t::BACKREF_INTERNAL, sm::label_instance("ext", "BACKREF_INTERNAL")}, - {extent_types_t::BACKREF_LEAF, sm::label_instance("ext", "BACKREF_LEAF")} + std::map> labels_by_ext { + {extent_types_t::ROOT, {sm::label_instance("ext", "ROOT")}}, + {extent_types_t::LADDR_INTERNAL, {sm::label_instance("ext", "LADDR_INTERNAL")}}, + {extent_types_t::LADDR_LEAF, {sm::label_instance("ext", "LADDR_LEAF")}}, + {extent_types_t::DINK_LADDR_LEAF, {sm::label_instance("ext", "DINK_LADDR_LEAF")}}, + {extent_types_t::ROOT_META, {sm::label_instance("ext", "ROOT_META")}}, + {extent_types_t::OMAP_INNER, {sm::label_instance("ext", "OMAP_INNER")}}, + {extent_types_t::OMAP_LEAF, {sm::label_instance("ext", "OMAP_LEAF")}}, + {extent_types_t::ONODE_BLOCK_STAGED, {sm::label_instance("ext", "ONODE_BLOCK_STAGED")}}, + {extent_types_t::COLL_BLOCK, {sm::label_instance("ext", "COLL_BLOCK")}}, + {extent_types_t::OBJECT_DATA_BLOCK, {sm::label_instance("ext", "OBJECT_DATA_BLOCK")}}, + {extent_types_t::RETIRED_PLACEHOLDER, {sm::label_instance("ext", "RETIRED_PLACEHOLDER")}}, + {extent_types_t::ALLOC_INFO, {sm::label_instance("ext", "ALLOC_INFO")}}, + {extent_types_t::JOURNAL_TAIL, {sm::label_instance("ext", "JOURNAL_TAIL")}}, + {extent_types_t::TEST_BLOCK, {sm::label_instance("ext", "TEST_BLOCK")}}, + {extent_types_t::TEST_BLOCK_PHYSICAL, {sm::label_instance("ext", "TEST_BLOCK_PHYSICAL")}}, + {extent_types_t::BACKREF_INTERNAL, {sm::label_instance("ext", "BACKREF_INTERNAL")}}, + {extent_types_t::BACKREF_LEAF, {sm::label_instance("ext", "BACKREF_LEAF")}} }; assert(labels_by_ext.size() == (std::size_t)extent_types_t::NONE); - + for (auto& [src, src_label] : labels_by_src) { + src_label.push_back(sm::label_instance("shard_store_index", std::to_string(store_index))); + } + for (auto& [ext, ext_label] : labels_by_ext) { + ext_label.push_back(sm::label_instance("shard_store_index", std::to_string(store_index))); + } /* * trans_created */ @@ -206,34 +212,40 @@ void Cache::register_metrics() [this] { return stats.access.get_cache_access(); }, - sm::description("total number of cache accesses") + sm::description("total number of cache accesses"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "cache_hit", [this] { return stats.access.get_cache_hit(); }, - sm::description("total number of cache hits") + sm::description("total number of cache hits"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "refresh_parent_total", cursor_stats.num_refresh_parent_total, - sm::description("total number of refreshed cursors") + sm::description("total number of refreshed cursors"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "refresh_invalid_parent", cursor_stats.num_refresh_invalid_parent, - sm::description("total number of refreshed cursors with invalid parents") + sm::description("total number of refreshed cursors with invalid parents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "refresh_unviewable_parent", cursor_stats.num_refresh_unviewable_parent, - sm::description("total number of refreshed cursors with unviewable parents") + sm::description("total number of refreshed cursors with unviewable parents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "refresh_modified_viewable_parent", cursor_stats.num_refresh_modified_viewable_parent, - sm::description("total number of refreshed cursors with viewable but modified parents") + sm::description("total number of refreshed cursors with viewable but modified parents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); @@ -257,6 +269,8 @@ void Cache::register_metrics() auto& efforts = get_by_src(stats.invalidated_efforts_by_src, src); for (auto& [ext, ext_label] : labels_by_ext) { auto& counter = get_by_ext(efforts.num_trans_invalidated, ext); + std::vector merged_labels = src_label; + merged_labels.insert(merged_labels.end(), ext_label.begin(), ext_label.end()); metrics.add_group( "cache", { @@ -264,7 +278,7 @@ void Cache::register_metrics() "trans_invalidated_by_extent", counter, sm::description("total number of transactions invalidated by extents"), - {src_label, ext_label} + merged_labels ), } ); @@ -273,6 +287,8 @@ void Cache::register_metrics() if (src == src_t::READ) { // read transaction won't have non-read efforts auto read_effort_label = effort_label("READ"); + std::vector merged_labels = src_label; + merged_labels.push_back(read_effort_label); metrics.add_group( "cache", { @@ -280,13 +296,13 @@ void Cache::register_metrics() "invalidated_extents", efforts.read.num, sm::description("extents of invalidated transactions"), - {src_label, read_effort_label} + merged_labels ), sm::make_counter( "invalidated_extent_bytes", efforts.read.bytes, sm::description("extent bytes of invalidated transactions"), - {src_label, read_effort_label} + merged_labels ), } ); @@ -309,6 +325,8 @@ void Cache::register_metrics() return efforts.fresh_ool_written; } }(); + std::vector merged_labels = src_label; + merged_labels.push_back(effort_label(effort_name)); metrics.add_group( "cache", { @@ -316,13 +334,13 @@ void Cache::register_metrics() "invalidated_extents", effort.num, sm::description("extents of invalidated transactions"), - {src_label, effort_label(effort_name)} + merged_labels ), sm::make_counter( "invalidated_extent_bytes", effort.bytes, sm::description("extent bytes of invalidated transactions"), - {src_label, effort_label(effort_name)} + merged_labels ), } ); @@ -428,7 +446,10 @@ void Cache::register_metrics() return efforts.fresh_ool_by_ext; } }(); + std::vector merged_labels = src_label; + merged_labels.push_back(effort_label(effort_name)); for (auto& [ext, ext_label] : labels_by_ext) { + merged_labels.insert(merged_labels.end(), ext_label.begin(), ext_label.end()); auto& effort = get_by_ext(effort_by_ext, ext); metrics.add_group( "cache", @@ -437,13 +458,13 @@ void Cache::register_metrics() "committed_extents", effort.num, sm::description("extents of committed transactions"), - {src_label, effort_label(effort_name), ext_label} + merged_labels ), sm::make_counter( "committed_extent_bytes", effort.bytes, sm::description("extent bytes of committed transactions"), - {src_label, effort_label(effort_name), ext_label} + merged_labels ), } ); @@ -452,6 +473,8 @@ void Cache::register_metrics() auto& delta_by_ext = efforts.delta_bytes_by_ext; for (auto& [ext, ext_label] : labels_by_ext) { + std::vector merged_labels = src_label; + merged_labels.insert(merged_labels.end(), ext_label.begin(), ext_label.end()); auto& value = get_by_ext(delta_by_ext, ext); metrics.add_group( "cache", @@ -460,7 +483,7 @@ void Cache::register_metrics() "committed_delta_bytes", value, sm::description("delta bytes of committed transactions"), - {src_label, ext_label} + merged_labels ), } ); @@ -474,17 +497,20 @@ void Cache::register_metrics() sm::make_counter( "trans_read_successful", stats.success_read_efforts.num_trans, - sm::description("total number of successful read transactions") + sm::description("total number of successful read transactions"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "successful_read_extents", stats.success_read_efforts.read.num, - sm::description("extents of successful read transactions") + sm::description("extents of successful read transactions"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "successful_read_extent_bytes", stats.success_read_efforts.read.bytes, - sm::description("extent bytes of successful read transactions") + sm::description("extent bytes of successful read transactions"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); @@ -503,31 +529,35 @@ void Cache::register_metrics() [this] { return extents_index.size(); }, - sm::description("total number of cached extents") + sm::description("total number of cached extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "cached_extent_bytes", [this] { return extents_index.get_bytes(); }, - sm::description("total bytes of cached extents") + sm::description("total bytes of cached extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "dirty_extents", [this] { return dirty.size(); }, - sm::description("total number of dirty extents") + sm::description("total number of dirty extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "dirty_extent_bytes", stats.dirty_bytes, - sm::description("total bytes of dirty extents") + sm::description("total bytes of dirty extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); - pinboard->register_metrics(); + pinboard->register_metrics(store_index); /** * tree stats @@ -537,7 +567,7 @@ void Cache::register_metrics() auto omap_label = tree_label("OMAP"); auto lba_label = tree_label("LBA"); auto backref_label = tree_label("BACKREF"); - auto register_tree_metrics = [&labels_by_src, &onode_label, &omap_label, this]( + auto register_tree_metrics = [&labels_by_src, &onode_label, &omap_label, store_index, this]( const sm::label_instance& tree_label, uint64_t& tree_depth, int64_t& tree_extents_num, @@ -550,13 +580,13 @@ void Cache::register_metrics() "tree_depth", tree_depth, sm::description("the depth of tree"), - {tree_label} + {tree_label, sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "tree_extents_num", tree_extents_num, sm::description("num of extents of the tree"), - {tree_label} + {tree_label, sm::label_instance("shard_store_index", std::to_string(store_index))} ) } ); @@ -573,6 +603,8 @@ void Cache::register_metrics() } auto& committed_efforts = get_by_src(committed_tree_efforts, src); auto& invalidated_efforts = get_by_src(invalidated_tree_efforts, src); + std::vector merged_labels = {tree_label}; // Copy src_label + merged_labels.insert(merged_labels.end(), src_label.begin(), src_label.end()); metrics.add_group( "cache", { @@ -580,37 +612,37 @@ void Cache::register_metrics() "tree_inserts_committed", committed_efforts.num_inserts, sm::description("total number of committed insert operations"), - {tree_label, src_label} + merged_labels ), sm::make_counter( "tree_erases_committed", committed_efforts.num_erases, sm::description("total number of committed erase operations"), - {tree_label, src_label} + merged_labels ), sm::make_counter( "tree_updates_committed", committed_efforts.num_updates, sm::description("total number of committed update operations"), - {tree_label, src_label} + merged_labels ), sm::make_counter( "tree_inserts_invalidated", invalidated_efforts.num_inserts, sm::description("total number of invalidated insert operations"), - {tree_label, src_label} + merged_labels ), sm::make_counter( "tree_erases_invalidated", invalidated_efforts.num_erases, sm::description("total number of invalidated erase operations"), - {tree_label, src_label} + merged_labels ), sm::make_counter( "tree_updates_invalidated", invalidated_efforts.num_updates, sm::description("total number of invalidated update operations"), - {tree_label, src_label} + merged_labels ), } ); @@ -675,7 +707,8 @@ void Cache::register_metrics() "trans_srcs_invalidated", stats.trans_conflicts_by_srcs[srcs_index - 1], sm::description("total number conflicted transactions by src pair"), - {srcs_label(oss.str())} + {srcs_label(oss.str()), + sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); @@ -695,7 +728,8 @@ void Cache::register_metrics() "trans_srcs_invalidated", stats.trans_conflicts_by_unknown[srcs_index - 1], sm::description("total number conflicted transactions by src pair"), - {srcs_label(oss.str())} + {srcs_label(oss.str()), + sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); @@ -712,24 +746,28 @@ void Cache::register_metrics() [this] { return stats.trim_rewrites.get_num_rewrites(); }, - sm::description("total number of rewrite-dirty extents") + sm::description("total number of rewrite-dirty extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "version_sum_dirty", stats.trim_rewrites.dirty_version, - sm::description("sum of the version from rewrite-dirty extents") + sm::description("sum of the version from rewrite-dirty extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "version_count_reclaim", [this] { return stats.reclaim_rewrites.get_num_rewrites(); }, - sm::description("total number of rewrite-reclaim extents") + sm::description("total number of rewrite-reclaim extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "version_sum_reclaim", stats.reclaim_rewrites.dirty_version, - sm::description("sum of the version from rewrite-reclaim extents") + sm::description("sum of the version from rewrite-reclaim extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index b5967d9f808..22c2231486a 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -106,7 +106,7 @@ class SegmentProvider; class Cache : public ExtentTransViewRetriever, public RetiredExtentPlaceholderInvalidater { public: - Cache(ExtentPlacementManager &epm); + Cache(ExtentPlacementManager &epm, unsigned int store_index); ~Cache(); cache_stats_t get_stats(bool report_detail, double seconds) const; @@ -1826,7 +1826,7 @@ private: } seastar::metrics::metric_group metrics; - void register_metrics(); + void register_metrics(unsigned int store_index); void apply_backref_mset( backref_entry_refs_t& backref_entries) { diff --git a/src/crimson/os/seastore/extent_pinboard.cc b/src/crimson/os/seastore/extent_pinboard.cc index 1d4cc84c800..483707eba9d 100644 --- a/src/crimson/os/seastore/extent_pinboard.cc +++ b/src/crimson/os/seastore/extent_pinboard.cc @@ -296,7 +296,7 @@ public: return lru.get_current_num_extents(); } - void register_metrics() final { + void register_metrics(unsigned int store_index) final { namespace sm = seastar::metrics; metrics.add_group( "cache", @@ -306,22 +306,26 @@ public: [this] { return get_current_size_bytes(); }, - sm::description("total bytes pinned by the lru") + sm::description("total bytes pinned by the lru"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "lru_num_extents", [this] { return get_current_num_extents(); }, - sm::description("total extents pinned by the lru") + sm::description("total extents pinned by the lru"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "lru_hit", hit, - sm::description("total count of the extents that are linked to lru when touching them") + sm::description("total count of the extents that are linked to lru when touching them"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "lru_miss", miss, - sm::description("total count of the extents that are not linked to lru when touching them") + sm::description("total count of the extents that are not linked to lru when touching them"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); @@ -505,7 +509,7 @@ public: return warm_in.get_current_num_extents() + hot.get_current_num_extents(); } - void register_metrics() final; + void register_metrics(unsigned int store_index) final; void get_stats( cache_stats_t &stats, @@ -802,7 +806,7 @@ void ExtentPinboardTwoQ::get_stats( last_hits = overall_hits; } -void ExtentPinboardTwoQ::register_metrics() { +void ExtentPinboardTwoQ::register_metrics(unsigned int store_index) { namespace sm = seastar::metrics; metrics.add_group( "cache", @@ -812,36 +816,42 @@ void ExtentPinboardTwoQ::register_metrics() { [this] { return warm_in.get_current_size_bytes(); }, - sm::description("total bytes pinned by the 2q warm_in queue") + sm::description("total bytes pinned by the 2q warm_in queue"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "2q_warm_in_num_extents", [this] { return warm_in.get_current_num_extents(); }, - sm::description("total extents pinned by the 2q warm_in queue") + sm::description("total extents pinned by the 2q warm_in queue"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "2q_hot_size_bytes", [this] { return hot.get_current_size_bytes(); }, - sm::description("total bytes pinned by the 2q hot queue") + sm::description("total bytes pinned by the 2q hot queue"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "2q_hot_num_extents", [this] { return hot.get_current_num_extents(); }, - sm::description("total extents pinned by the 2q hot queue") + sm::description("total extents pinned by the 2q hot queue"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "2q_hit", hit, - sm::description("total count of the extents that are linked to 2Q when touching them") + sm::description("total count of the extents that are linked to 2Q when touching them"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "2q_miss", miss, - sm::description("total count of the extents that are not linked to 2Q when touching them") + sm::description("total count of the extents that are not linked to 2Q when touching them"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); diff --git a/src/crimson/os/seastore/extent_pinboard.h b/src/crimson/os/seastore/extent_pinboard.h index 9f6935d6cef..2a891d5c3b5 100644 --- a/src/crimson/os/seastore/extent_pinboard.h +++ b/src/crimson/os/seastore/extent_pinboard.h @@ -10,7 +10,7 @@ namespace crimson::os::seastore { struct ExtentPinboard { virtual ~ExtentPinboard() = default; - virtual void register_metrics() = 0; + virtual void register_metrics(unsigned int store_index) = 0; virtual void move_to_top( CachedExtent &extent, const Transaction::src_t *p_src, diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 50bf7dddcdc..db753e630de 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -12,11 +12,13 @@ SET_SUBSYS(seastore_epm); namespace crimson::os::seastore { SegmentedOolWriter::SegmentedOolWriter( + unsigned int store_index, data_category_t category, rewrite_gen_t gen, SegmentProvider& sp, SegmentSeqAllocator &ssa) - : segment_allocator(nullptr, category, gen, sp, ssa), + : store_index(store_index), + segment_allocator(nullptr, category, gen, sp, ssa), record_submitter(crimson::common::get_conf( "seastore_journal_iodepth_limit"), crimson::common::get_conf( @@ -215,7 +217,7 @@ void ExtentPlacementManager::init( // DATA data_writers_by_gen.resize(num_writers, nullptr); for (rewrite_gen_t gen = OOL_GENERATION; gen < hot_tier_generations; ++gen) { - writer_refs.emplace_back(std::make_unique( + writer_refs.emplace_back(std::make_unique(store_index, data_category_t::DATA, gen, *segment_cleaner, *ool_segment_seq_allocator)); data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); @@ -224,7 +226,7 @@ void ExtentPlacementManager::init( // METADATA md_writers_by_gen.resize(num_writers, {}); for (rewrite_gen_t gen = OOL_GENERATION; gen < hot_tier_generations; ++gen) { - writer_refs.emplace_back(std::make_unique( + writer_refs.emplace_back(std::make_unique(store_index, data_category_t::METADATA, gen, *segment_cleaner, *ool_segment_seq_allocator)); md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); @@ -256,14 +258,14 @@ void ExtentPlacementManager::init( if (cold_segment_cleaner) { // Cold DATA Segments for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { - writer_refs.emplace_back(std::make_unique( + writer_refs.emplace_back(std::make_unique(store_index, data_category_t::DATA, gen, *cold_segment_cleaner, *ool_segment_seq_allocator)); data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { // Cold METADATA Segments - writer_refs.emplace_back(std::make_unique( + writer_refs.emplace_back(std::make_unique(store_index, data_category_t::METADATA, gen, *cold_segment_cleaner, *ool_segment_seq_allocator)); md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); @@ -597,14 +599,14 @@ void ExtentPlacementManager::BackgroundProcess::log_state(const char *caller) co } } -ExtentPlacementManager::mount_ret ExtentPlacementManager::BackgroundProcess::mount() { +ExtentPlacementManager::mount_ret ExtentPlacementManager::BackgroundProcess::mount(unsigned int store_index) { LOG_PREFIX(BackgroundProcess::mount); DEBUG("start"); ceph_assert(state == state_t::STOP); state = state_t::MOUNT; trimmer->reset(); stats = {}; - register_metrics(); + register_metrics(store_index); DEBUG("mounting main cleaner"); co_await main_cleaner->mount(); if (has_cold_tier()) { @@ -1006,26 +1008,34 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() } } -void ExtentPlacementManager::BackgroundProcess::register_metrics() +void ExtentPlacementManager::BackgroundProcess::register_metrics(unsigned int store_index) { namespace sm = seastar::metrics; metrics.add_group("background_process", { sm::make_counter("io_count", stats.io_count, - sm::description("the sum of IOs")), + sm::description("the sum of IOs"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("io_blocked_count", stats.io_blocked_count, - sm::description("IOs that are blocked by gc")), + sm::description("IOs that are blocked by gc"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("io_blocked_count_trim", stats.io_blocked_count_trim, - sm::description("IOs that are blocked by trimming")), + sm::description("IOs that are blocked by trimming"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("io_retried_blocked_count_clean", stats.io_blocked_count_clean, - sm::description("Retried IOs that are blocked by cleaning")), + sm::description("Retried IOs that are blocked by cleaning"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("io_retried_blocked_count_trim", stats.io_blocked_count_trim, - sm::description("Retried IOs that are blocked by trimming")), + sm::description("Retried IOs that are blocked by trimming"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("io_blocked_count_clean", stats.io_blocked_count_clean, - sm::description("IOs that are blocked by cleaning")), + sm::description("IOs that are blocked by cleaning"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("io_blocked_sum", stats.io_blocked_sum, - sm::description("the sum of blocking IOs")), + sm::description("the sum of blocking IOs"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), sm::make_counter("io_blocked_time", stats.io_blocked_time, - sm::description("the sum of the time(ms) in which IOs are blocked")) + sm::description("the sum of the time(ms) in which IOs are blocked"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}) }); } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index a5a6846e856..9937272fbd9 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -69,7 +69,8 @@ using ExtentOolWriterRef = std::unique_ptr; */ class SegmentedOolWriter : public ExtentOolWriter { public: - SegmentedOolWriter(data_category_t category, + SegmentedOolWriter(unsigned int store_index, + data_category_t category, rewrite_gen_t gen, SegmentProvider &sp, SegmentSeqAllocator &ssa); @@ -83,7 +84,7 @@ public: } open_ertr::future<> open() final { - return record_submitter.open(false).discard_result(); + return record_submitter.open(store_index, false).discard_result(); } alloc_write_iertr::future<> alloc_write_ool_extents( @@ -122,6 +123,7 @@ private: std::list &&extents, bool with_atomic_roll_segment=false); + unsigned int store_index; journal::SegmentAllocator segment_allocator; journal::RecordSubmitter record_submitter; seastar::gate write_guard; @@ -269,10 +271,12 @@ class ExtentPlacementManager { public: ExtentPlacementManager( rewrite_gen_t hot_tier_generations, - rewrite_gen_t cold_tier_generations) + rewrite_gen_t cold_tier_generations, + unsigned int store_index) : hot_tier_generations(hot_tier_generations), cold_tier_generations(cold_tier_generations), dynamic_max_rewrite_generation(cold_tier_generations), + store_index(store_index), ool_segment_seq_allocator( std::make_unique(segment_type_t::OOL)), max_data_allocation_size(crimson::common::get_conf( @@ -331,7 +335,7 @@ public: crimson::ct_error::input_output_error>; using mount_ret = mount_ertr::future<>; mount_ret mount() { - return background_process.mount(); + return background_process.mount(store_index); } using open_ertr = ExtentOolWriter::open_ertr; @@ -771,7 +775,7 @@ private: return stat; } - ExtentPlacementManager::mount_ret mount(); + ExtentPlacementManager::mount_ret mount(unsigned int store_index); void start_scan_space() { ceph_assert(state == state_t::MOUNT); @@ -1080,7 +1084,7 @@ private: seastar::future<> do_background_cycle(); - void register_metrics(); + void register_metrics(unsigned int store_index); struct { uint64_t io_blocking_num = 0; @@ -1128,6 +1132,7 @@ private: const rewrite_gen_t cold_tier_generations = NULL_GENERATION; rewrite_gen_t dynamic_max_rewrite_generation = NULL_GENERATION; BackgroundProcess background_process; + unsigned int store_index = 0; // TODO: drop once paddr->journal_seq_t is introduced SegmentSeqAllocatorRef ool_segment_seq_allocator; extent_len_t max_data_allocation_size = 0; diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index 91323de333b..dc098f58bbd 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -8,18 +8,20 @@ namespace crimson::os::seastore::journal { JournalRef make_segmented( + unsigned int store_index, SegmentProvider &provider, JournalTrimmer &trimmer) { - return std::make_unique(provider, trimmer); + return std::make_unique(store_index, provider, trimmer); } JournalRef make_circularbounded( + unsigned int store_index, JournalTrimmer &trimmer, crimson::os::seastore::random_block_device::RBMDevice* device, std::string path) { - return std::make_unique(trimmer, device, path); + return std::make_unique(store_index, trimmer, device, path); } } diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index 3e9beaa248d..ef28436c5e3 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -112,10 +112,12 @@ using JournalRef = std::unique_ptr; namespace journal { JournalRef make_segmented( + unsigned int store_index, SegmentProvider &provider, JournalTrimmer &trimmer); JournalRef make_circularbounded( + unsigned int store_index, JournalTrimmer &trimmer, crimson::os::seastore::random_block_device::RBMDevice* device, std::string path); diff --git a/src/crimson/os/seastore/journal/circular_bounded_journal.cc b/src/crimson/os/seastore/journal/circular_bounded_journal.cc index a13be09148e..115db64e364 100644 --- a/src/crimson/os/seastore/journal/circular_bounded_journal.cc +++ b/src/crimson/os/seastore/journal/circular_bounded_journal.cc @@ -15,10 +15,12 @@ SET_SUBSYS(seastore_journal); namespace crimson::os::seastore::journal { CircularBoundedJournal::CircularBoundedJournal( + unsigned int store_index, JournalTrimmer &trimmer, RBMDevice* device, const std::string &path) - : trimmer(trimmer), path(path), + : store_index(store_index), + trimmer(trimmer), path(path), cjs(device), record_submitter(crimson::common::get_conf( "seastore_journal_iodepth_limit"), @@ -36,7 +38,7 @@ CircularBoundedJournal::CircularBoundedJournal( CircularBoundedJournal::open_for_mkfs_ret CircularBoundedJournal::open_for_mkfs() { - return record_submitter.open(true + return record_submitter.open(store_index, true ).safe_then([this](auto ret) { return open_for_mkfs_ret( open_for_mkfs_ertr::ready_future_marker{}, @@ -47,7 +49,7 @@ CircularBoundedJournal::open_for_mkfs() CircularBoundedJournal::open_for_mount_ret CircularBoundedJournal::open_for_mount() { - return record_submitter.open(false + return record_submitter.open(store_index, false ).safe_then([this](auto ret) { return open_for_mount_ret( open_for_mount_ertr::ready_future_marker{}, diff --git a/src/crimson/os/seastore/journal/circular_bounded_journal.h b/src/crimson/os/seastore/journal/circular_bounded_journal.h index cd8c7a1a84c..925fe171d9b 100644 --- a/src/crimson/os/seastore/journal/circular_bounded_journal.h +++ b/src/crimson/os/seastore/journal/circular_bounded_journal.h @@ -55,6 +55,7 @@ constexpr uint64_t DEFAULT_BLOCK_SIZE = 4096; class CircularBoundedJournal : public Journal, RecordScanner { public: CircularBoundedJournal( + unsigned int store_index, JournalTrimmer &trimmer, RBMDevice* device, const std::string &path); ~CircularBoundedJournal() {} @@ -203,6 +204,7 @@ public: } private: + unsigned int store_index; JournalTrimmer &trimmer; std::string path; WritePipeline *write_pipeline = nullptr; diff --git a/src/crimson/os/seastore/journal/record_submitter.cc b/src/crimson/os/seastore/journal/record_submitter.cc index acff722f07c..3b0f91c5195 100644 --- a/src/crimson/os/seastore/journal/record_submitter.cc +++ b/src/crimson/os/seastore/journal/record_submitter.cc @@ -389,10 +389,10 @@ RecordSubmitter::submit( } RecordSubmitter::open_ret -RecordSubmitter::open(bool is_mkfs) +RecordSubmitter::open(unsigned int store_index, bool is_mkfs) { return journal_allocator.open(is_mkfs - ).safe_then([this](journal_seq_t ret) { + ).safe_then([this, store_index](journal_seq_t ret) { LOG_PREFIX(RecordSubmitter::open); DEBUG("{} register metrics", get_name()); stats = {}; @@ -400,6 +400,8 @@ RecordSubmitter::open(bool is_mkfs) namespace sm = seastar::metrics; std::vector label_instances; label_instances.push_back(sm::label_instance("submitter", get_name())); + label_instances.push_back(sm::label_instance("shard_store_index", std::to_string(store_index))); + metrics.add_group( "journal", { diff --git a/src/crimson/os/seastore/journal/record_submitter.h b/src/crimson/os/seastore/journal/record_submitter.h index 8f085448279..0140c38c6ab 100644 --- a/src/crimson/os/seastore/journal/record_submitter.h +++ b/src/crimson/os/seastore/journal/record_submitter.h @@ -289,7 +289,7 @@ public: // open for write, generate the correct print name, and register metrics using open_ertr = base_ertr; using open_ret = open_ertr::future; - open_ret open(bool is_mkfs); + open_ret open(unsigned int store_index, bool is_mkfs); using close_ertr = base_ertr; close_ertr::future<> close(); diff --git a/src/crimson/os/seastore/journal/segmented_journal.cc b/src/crimson/os/seastore/journal/segmented_journal.cc index 5ed8595aebf..9c4fc61bbce 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.cc +++ b/src/crimson/os/seastore/journal/segmented_journal.cc @@ -27,9 +27,11 @@ SET_SUBSYS(seastore_journal); namespace crimson::os::seastore::journal { SegmentedJournal::SegmentedJournal( + unsigned int store_index, SegmentProvider &segment_provider, JournalTrimmer &trimmer) - : segment_seq_allocator( + : store_index(store_index), + segment_seq_allocator( new SegmentSeqAllocator(segment_type_t::JOURNAL)), journal_segment_allocator(&trimmer, data_category_t::METADATA, @@ -53,13 +55,13 @@ SegmentedJournal::SegmentedJournal( SegmentedJournal::open_for_mkfs_ret SegmentedJournal::open_for_mkfs() { - return record_submitter.open(true); + return record_submitter.open(store_index, true); } SegmentedJournal::open_for_mount_ret SegmentedJournal::open_for_mount() { - return record_submitter.open(false); + return record_submitter.open(store_index, false); } SegmentedJournal::close_ertr::future<> SegmentedJournal::close() diff --git a/src/crimson/os/seastore/journal/segmented_journal.h b/src/crimson/os/seastore/journal/segmented_journal.h index 6e0cb0ca5dd..ef8d3e0e1bf 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.h +++ b/src/crimson/os/seastore/journal/segmented_journal.h @@ -26,6 +26,7 @@ namespace crimson::os::seastore::journal { class SegmentedJournal : public Journal { public: SegmentedJournal( + unsigned int store_index, SegmentProvider &segment_provider, JournalTrimmer &trimmer); ~SegmentedJournal() {} @@ -73,6 +74,7 @@ private: OrderingHandle &handle, on_submission_func_t &&on_submission); + unsigned int store_index; SegmentSeqAllocatorRef segment_seq_allocator; SegmentAllocator journal_segment_allocator; RecordSubmitter record_submitter; diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index db492e83247..865b621f1ae 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -1038,7 +1038,7 @@ BtreeLBAManager::complete_indirect_lba_mapping( }); } -void BtreeLBAManager::register_metrics() +void BtreeLBAManager::register_metrics(unsigned int store_index) { LOG_PREFIX(BtreeLBAManager::register_metrics); DEBUG("start"); @@ -1050,12 +1050,14 @@ void BtreeLBAManager::register_metrics() sm::make_counter( "alloc_extents", stats.num_alloc_extents, - sm::description("total number of lba alloc_extent operations") + sm::description("total number of lba alloc_extent operations"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), sm::make_counter( "alloc_extents_iter_nexts", stats.num_alloc_extents_iter_nexts, - sm::description("total number of iterator next operations during extent allocation") + sm::description("total number of iterator next operations during extent allocation"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} ), } ); diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.h b/src/crimson/os/seastore/lba/btree_lba_manager.h index c34e16062a7..3ede2b53c4a 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.h +++ b/src/crimson/os/seastore/lba/btree_lba_manager.h @@ -53,10 +53,10 @@ using LBABtree = FixedKVBtree< */ class BtreeLBAManager : public LBAManager { public: - BtreeLBAManager(Cache &cache) + BtreeLBAManager(Cache &cache, unsigned int store_index) : cache(cache) { - register_metrics(); + register_metrics(store_index); } mkfs_ret mkfs( @@ -433,7 +433,7 @@ private: } seastar::metrics::metric_group metrics; - void register_metrics(); + void register_metrics(unsigned int store_index); struct update_mapping_ret_bare_t { update_mapping_ret_bare_t() diff --git a/src/crimson/os/seastore/lba_manager.cc b/src/crimson/os/seastore/lba_manager.cc index 49941635325..fd7118c2dbe 100644 --- a/src/crimson/os/seastore/lba_manager.cc +++ b/src/crimson/os/seastore/lba_manager.cc @@ -6,8 +6,8 @@ namespace crimson::os::seastore { -LBAManagerRef lba::create_lba_manager(Cache &cache) { - return LBAManagerRef(new lba::BtreeLBAManager(cache)); +LBAManagerRef lba::create_lba_manager(Cache &cache, unsigned int store_index) { + return LBAManagerRef(new lba::BtreeLBAManager(cache, store_index)); } } diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h index 49182d6edcd..5f912c4d9db 100644 --- a/src/crimson/os/seastore/lba_manager.h +++ b/src/crimson/os/seastore/lba_manager.h @@ -359,7 +359,7 @@ using LBAManagerRef = std::unique_ptr; class Cache; namespace lba { -LBAManagerRef create_lba_manager(Cache &cache); +LBAManagerRef create_lba_manager(Cache &cache, unsigned int store_index); } } diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 7baea2cd82c..6da30c04a0f 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -996,6 +996,7 @@ TransactionManagerRef make_transaction_manager( Device *primary_device, const std::vector &secondary_devices, shard_stats_t& shard_stats, + unsigned int store_index, bool is_test) { LOG_PREFIX(make_transaction_manager); @@ -1004,9 +1005,9 @@ TransactionManagerRef make_transaction_manager( rewrite_gen_t cold_tier_generations = crimson::common::get_conf( "seastore_cold_tier_generations"); auto epm = std::make_unique( - hot_tier_generations, cold_tier_generations); - auto cache = std::make_unique(*epm); - auto lba_manager = lba::create_lba_manager(*cache); + hot_tier_generations, cold_tier_generations, store_index); + auto cache = std::make_unique(*epm, store_index); + auto lba_manager = lba::create_lba_manager(*cache, store_index); auto sms = std::make_unique(); auto rbs = std::make_unique(); auto backref_manager = create_backref_manager(*cache); @@ -1080,6 +1081,7 @@ TransactionManagerRef make_transaction_manager( bool pure_rbm_backend = (p_backend_type == backend_type_t::RANDOM_BLOCK) && !cold_sms; auto journal_trimmer = JournalTrimmerImpl::create( + store_index, *backref_manager, trimmer_config, backend_type, roll_start, roll_size, !pure_rbm_backend); @@ -1091,6 +1093,7 @@ TransactionManagerRef make_transaction_manager( if (cold_sms) { cold_segment_cleaner = SegmentCleaner::create( + store_index, cleaner_config, std::move(cold_sms), *backref_manager, @@ -1108,6 +1111,7 @@ TransactionManagerRef make_transaction_manager( if (backend_type == backend_type_t::SEGMENTED) { cleaner = SegmentCleaner::create( + store_index, cleaner_config, std::move(sms), *backref_manager, @@ -1121,15 +1125,18 @@ TransactionManagerRef make_transaction_manager( } segment_cleaner->set_journal_trimmer(*journal_trimmer); journal = journal::make_segmented( + store_index, *segment_cleaner, *journal_trimmer); } else { cleaner = RBMCleaner::create( + store_index, std::move(rbs), *backref_manager, *lba_manager, cleaner_is_detailed); journal = journal::make_circularbounded( + store_index, *journal_trimmer, static_cast(primary_device), ""); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 1ba7599051f..7f8c35180f5 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -1565,5 +1565,6 @@ TransactionManagerRef make_transaction_manager( Device *primary_device, const std::vector &secondary_devices, shard_stats_t& shard_stats, + unsigned int store_index, bool is_test); }