From: chunmei-liu Date: Tue, 11 May 2021 01:28:56 +0000 (-0700) Subject: crimson/seastore: add perfcoutners in seastore X-Git-Tag: v17.1.0~1978^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4c0f3127b195df3538109ca9ca869a07219013e6;p=ceph.git crimson/seastore: add perfcoutners in seastore Signed-off-by: chunmei-liu --- diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 05d72771c29d1..cf9a6cb02b7de 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -29,7 +29,9 @@ using crimson::common::local_conf; namespace crimson::os::seastore { -SeaStore::~SeaStore() {} +SeaStore::~SeaStore() { + perf_service->remove_from_collection(); +} struct SeastoreCollection final : public FuturizedCollection { template @@ -1085,9 +1087,11 @@ std::unique_ptr make_seastore( segment_manager::block::BlockSegmentManager >(device + "/block"); + PerfServiceRef perf_service = PerfServiceRef(new PerfService()); + auto segment_cleaner = std::make_unique( SegmentCleaner::config_t::get_default(), - false /* detailed */); + perf_service->get_counters(), false /* detailed */); auto journal = std::make_unique(*sm); auto cache = std::make_unique(*sm); @@ -1100,14 +1104,16 @@ std::unique_ptr make_seastore( std::move(segment_cleaner), std::move(journal), std::move(cache), - std::move(lba_manager)); + std::move(lba_manager), + perf_service->get_counters()); auto cm = std::make_unique(*tm); return std::make_unique( std::move(sm), std::move(tm), std::move(cm), - std::make_unique(*tm)); + std::make_unique(*tm), + std::move(perf_service)); } } diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index c7698efbf3d5b..f340e4015670a 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -22,6 +22,7 @@ #include "crimson/os/seastore/onode_manager.h" #include "crimson/os/seastore/omap_manager.h" #include "crimson/os/seastore/collection_manager.h" +#include "crimson/os/seastore/seastore_perf_counters.h" namespace crimson::os::seastore { @@ -37,11 +38,15 @@ public: SegmentManagerRef sm, TransactionManagerRef tm, CollectionManagerRef cm, - OnodeManagerRef om + OnodeManagerRef om, + PerfServiceRef p_service ) : segment_manager(std::move(sm)), transaction_manager(std::move(tm)), collection_manager(std::move(cm)), - onode_manager(std::move(om)) {} + onode_manager(std::move(om)), + perf_service(std::move(p_service)) { + perf_service->add_to_collection(); + } ~SeaStore(); @@ -246,6 +251,7 @@ private: TransactionManagerRef transaction_manager; CollectionManagerRef collection_manager; OnodeManagerRef onode_manager; + PerfServiceRef perf_service; using tm_ertr = TransactionManager::base_ertr; using tm_ret = tm_ertr::future<>; diff --git a/src/crimson/os/seastore/seastore_perf_counters.h b/src/crimson/os/seastore/seastore_perf_counters.h new file mode 100644 index 0000000000000..cfeae4ebe380f --- /dev/null +++ b/src/crimson/os/seastore/seastore_perf_counters.h @@ -0,0 +1,77 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#pragma once + +#include "crimson/common/perf_counters_collection.h" + +namespace crimson::os::seastore { + +using PerfCountersRef = std::unique_ptr; +enum { + seastore_perfcounters_first = 30000, + ss_tm_extents_mutated_total, + ss_tm_extents_mutated_bytes, + ss_tm_extents_retired_total, + ss_tm_extents_retired_bytes, + ss_tm_extents_allocated_total, + ss_tm_extents_allocated_bytes, + ss_sm_bytes_written, + ss_sm_segments_released, + ss_sm_bytes_released, + seastore_perfcounters_last, +}; + +class PerfService { + PerfCountersRef build_seastore_perf() { + std::string name = fmt::format("seastore::shard-{}", seastar::this_shard_id()); + PerfCountersBuilder ss_plb(nullptr, name, seastore_perfcounters_first, + seastore_perfcounters_last); + ss_plb.add_u64_counter( + ss_tm_extents_mutated_total, "extents_mutated_num", + "the number of extents which have been mutated"); + ss_plb.add_u64_counter( + ss_tm_extents_mutated_bytes, "extents_mutated_bytes", + "the total bytes of extents which have been mutated"); + ss_plb.add_u64_counter( + ss_tm_extents_retired_total, "extents_retired_num", + "the number of extents which have been retired"); + ss_plb.add_u64_counter( + ss_tm_extents_retired_bytes, "extents_retired_bytes", + "the total bytes of extents which have been retired"); + ss_plb.add_u64_counter( + ss_tm_extents_allocated_total, "extents_allocated_num", + "the number of extents which have been allocated"); + ss_plb.add_u64_counter( + ss_tm_extents_allocated_bytes, "extents_allocated_bytes", + "the total bytes of extents which have been allocated"); + ss_plb.add_u64_counter( + ss_sm_bytes_written, "sm_bytes_written", + "the number of bytes written by segmentmanager"); + ss_plb.add_u64_counter( + ss_sm_segments_released, "sm_segments_released", + "the number of segments released by segmentmanager"); + ss_plb.add_u64_counter( + ss_sm_bytes_released, "sm_bytes_released", + "the number of bytes released by segmentmanager"); + return PerfCountersRef(ss_plb.create_perf_counters()); + } + + PerfCountersRef ss_perf = nullptr; +public: + PerfService() + : ss_perf(build_seastore_perf()) {} + + ~PerfService() {} + + PerfCounters& get_counters() { + return *ss_perf; + } + void add_to_collection() { + crimson::common::local_perf_coll().get_perf_collection()->add(ss_perf.get()); + } + void remove_from_collection() { + crimson::common::local_perf_coll().get_perf_collection()->remove(ss_perf.get()); + } +}; +using PerfServiceRef = std::unique_ptr; +} diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index db3b8bf87701d..d3947b0913eaa 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -13,6 +13,7 @@ #include "crimson/os/seastore/cached_extent.h" #include "crimson/os/seastore/journal.h" #include "crimson/os/seastore/seastore_types.h" +#include "crimson/os/seastore/seastore_perf_counters.h" #include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/transaction.h" @@ -348,6 +349,7 @@ private: size_t empty_segments; int64_t used_bytes = 0; bool init_complete = false; + PerfCounters &sc_perf; /// target journal_tail for next fresh segment journal_seq_t journal_tail_target; @@ -364,9 +366,10 @@ private: std::optional> blocked_io_wake; public: - SegmentCleaner(config_t config, bool detailed = false) + SegmentCleaner(config_t config, PerfCounters &perf, bool detailed = false) : detailed(detailed), config(config), + sc_perf(perf), gc_process(*this) {} void mount(SegmentManager &sm) { @@ -447,6 +450,7 @@ public: } void mark_segment_released(segment_id_t segment) { + sc_perf.inc(ss_sm_segments_released); return mark_empty(segment); } diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index e0f6860c1c222..2977850f0468f 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -16,12 +16,14 @@ TransactionManager::TransactionManager( SegmentCleanerRef _segment_cleaner, JournalRef _journal, CacheRef _cache, - LBAManagerRef _lba_manager) + LBAManagerRef _lba_manager, + PerfCounters &perf) : segment_manager(_segment_manager), segment_cleaner(std::move(_segment_cleaner)), cache(std::move(_cache)), lba_manager(std::move(_lba_manager)), - journal(std::move(_journal)) + journal(std::move(_journal)), + tm_perf(perf) { segment_cleaner->set_extent_callback(this); journal->set_write_pipeline(&write_pipeline); @@ -156,6 +158,8 @@ TransactionManager::ref_ret TransactionManager::dec_ref( t, *ref); cache->retire_extent(t, ref); + tm_perf.inc(ss_tm_extents_retired_total); + tm_perf.inc(ss_tm_extents_retired_bytes, ref->get_length()); } return ret.refcount; }); @@ -172,7 +176,9 @@ TransactionManager::ref_ret TransactionManager::dec_ref( DEBUGT("offset {} refcount 0", t, offset); return cache->retire_extent( t, result.addr, result.length - ).safe_then([] { + ).safe_then([result, this] { + tm_perf.inc(ss_tm_extents_retired_total); + tm_perf.inc(ss_tm_extents_retired_bytes, result.length); return ref_ret( ref_ertr::ready_future_marker{}, 0); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 61b85e7023245..56407c13fbcae 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -23,6 +23,7 @@ #include "crimson/os/seastore/logging.h" #include "crimson/os/seastore/segment_cleaner.h" #include "crimson/os/seastore/seastore_types.h" +#include "crimson/os/seastore/seastore_perf_counters.h" #include "crimson/os/seastore/cache.h" #include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/lba_manager.h" @@ -68,7 +69,8 @@ public: SegmentCleanerRef segment_cleaner, JournalRef journal, CacheRef cache, - LBAManagerRef lba_manager); + LBAManagerRef lba_manager, + PerfCounters &perf); /// Writes initial metadata to disk using mkfs_ertr = crimson::errorator< @@ -188,6 +190,8 @@ public: auto ret = cache->duplicate_for_write( t, ref)->cast(); + tm_perf.inc(ss_tm_extents_mutated_total); + tm_perf.inc(ss_tm_extents_mutated_bytes, ret->get_length()); if (!ret->has_pin()) { DEBUGT( "duplicating {} for write: {}", @@ -258,8 +262,10 @@ public: hint, len, ext->get_paddr() - ).safe_then([ext=std::move(ext)](auto &&ref) mutable { + ).safe_then([ext=std::move(ext), len, this](auto &&ref) mutable { ext->set_pin(std::move(ref)); + tm_perf.inc(ss_tm_extents_allocated_total); + tm_perf.inc(ss_tm_extents_allocated_bytes, len); return alloc_extent_ertr::make_ready_future>( std::move(ext)); }); @@ -489,6 +495,7 @@ private: CacheRef cache; LBAManagerRef lba_manager; JournalRef journal; + PerfCounters &tm_perf; WritePipeline write_pipeline; diff --git a/src/crimson/tools/store_nbd/store-nbd.cc b/src/crimson/tools/store_nbd/store-nbd.cc index e5286d25bc161..a3f228ec5acad 100644 --- a/src/crimson/tools/store_nbd/store-nbd.cc +++ b/src/crimson/tools/store_nbd/store-nbd.cc @@ -40,7 +40,7 @@ #include "crimson/common/log.h" #include "crimson/common/config_proxy.h" -#include "crimson/common/perf_counters_collection.h" +#include "crimson/os/seastore/seastore_perf_counters.h" #include "test/crimson/seastar_runner.h" @@ -306,22 +306,24 @@ int main(int argc, char** argv) } sc.run([=] { - return crimson::common::sharded_conf( - ).start(EntityName{}, string_view{"ceph"} - ).then([=] { - auto backend = get_backend(backend_config); - return seastar::do_with( - NBDHandler(*backend, nbd_config), - std::move(backend), - [](auto &nbd, auto &backend) { - return backend->mount( - ).then([&] { - logger().debug("Running nbd server..."); - return nbd.run(); - }).then([&] { - return backend->close(); - }); + return crimson::common::sharded_perf_coll().start().then([=] { + return crimson::common::sharded_conf( + ).start(EntityName{}, string_view{"ceph"} + ).then([=] { + auto backend = get_backend(backend_config); + return seastar::do_with( + NBDHandler(*backend, nbd_config), + std::move(backend), + [](auto &nbd, auto &backend) { + return backend->mount( + ).then([&] { + logger().debug("Running nbd server..."); + return nbd.run(); + }).then([&] { + return backend->close(); + }); }); + }); }).then([=] { return crimson::common::sharded_perf_coll().stop().then([] { return crimson::common::sharded_conf().stop(); diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc index b45bd0d52b81e..9b69e68b81cf0 100644 --- a/src/crimson/tools/store_nbd/tm_driver.cc +++ b/src/crimson/tools/store_nbd/tm_driver.cc @@ -133,7 +133,7 @@ void TMDriver::init() { auto segment_cleaner = std::make_unique( SegmentCleaner::config_t::get_default(), - false /* detailed */); + perf_service->get_counters(), false /* detailed */); segment_cleaner->mount(*segment_manager); auto journal = std::make_unique(*segment_manager); auto cache = std::make_unique(*segment_manager); @@ -146,7 +146,8 @@ void TMDriver::init() std::move(segment_cleaner), std::move(journal), std::move(cache), - std::move(lba_manager)); + std::move(lba_manager), + perf_service->get_counters()); } void TMDriver::clear() @@ -162,6 +163,7 @@ size_t TMDriver::get_size() const seastar::future<> TMDriver::mkfs() { assert(config.path); + perf_service->add_to_collection(); segment_manager = std::make_unique< segment_manager::block::BlockSegmentManager >(*config.path); @@ -184,6 +186,7 @@ seastar::future<> TMDriver::mkfs() logger().debug("sm close"); return segment_manager->close(); }).safe_then([this] { + perf_service->remove_from_collection(); clear(); logger().debug("mkfs complete"); return TransactionManager::mkfs_ertr::now(); diff --git a/src/crimson/tools/store_nbd/tm_driver.h b/src/crimson/tools/store_nbd/tm_driver.h index 1643281339f42..d5be3c228f02b 100644 --- a/src/crimson/tools/store_nbd/tm_driver.h +++ b/src/crimson/tools/store_nbd/tm_driver.h @@ -4,6 +4,7 @@ #include "block_driver.h" #include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/seastore_perf_counters.h" #include "crimson/os/seastore/segment_cleaner.h" #include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/segment_manager/block.h" @@ -41,6 +42,9 @@ private: using TransactionManager = crimson::os::seastore::TransactionManager; std::unique_ptr tm; + using PerfServiceRef = crimson::os::seastore::PerfServiceRef; + using PerfService = crimson::os::seastore::PerfService; + PerfServiceRef perf_service = PerfServiceRef(new PerfService()); seastar::future<> mkfs(); void init(); diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index 93a8b222a3fd2..1a0d65e5e802d 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -60,7 +60,6 @@ struct transaction_manager_test_t : transaction_manager_test_t() : gen(rd()) { - init(); } laddr_t get_random_laddr(size_t block_size, laddr_t limit) { diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index ce99b76796c2d..07d8061f306fb 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -10,6 +10,7 @@ #include "crimson/os/seastore/transaction_manager.h" #include "crimson/os/seastore/segment_manager/ephemeral.h" #include "crimson/os/seastore/seastore.h" +#include "crimson/os/seastore/seastore_perf_counters.h" #include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/collection_manager/flat_collection_manager.h" #include "crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h" @@ -69,11 +70,11 @@ protected: }; auto get_transaction_manager( - SegmentManager &segment_manager + SegmentManager &segment_manager, PerfCounters &perf ) { auto segment_cleaner = std::make_unique( SegmentCleaner::config_t::get_default(), - true); + perf, true); auto journal = std::make_unique(segment_manager); auto cache = std::make_unique(segment_manager); auto lba_manager = lba_manager::create_lba_manager(segment_manager, *cache); @@ -85,20 +86,23 @@ auto get_transaction_manager( std::move(segment_cleaner), std::move(journal), std::move(cache), - std::move(lba_manager)); + std::move(lba_manager), + perf); return ret; } auto get_seastore( SegmentManagerRef sm ) { - auto tm = get_transaction_manager(*sm); + PerfServiceRef perf_service = PerfServiceRef(new PerfService()); + auto tm = get_transaction_manager(*sm, perf_service->get_counters()); auto cm = std::make_unique(*tm); return std::make_unique( std::move(sm), std::move(tm), std::move(cm), - std::make_unique(*tm)); + std::make_unique(*tm), + std::move(perf_service)); } @@ -107,11 +111,13 @@ protected: std::unique_ptr tm; LBAManager *lba_manager; SegmentCleaner *segment_cleaner; + PerfServiceRef perf_service = PerfServiceRef(new PerfService()); TMTestState() : EphemeralTestState() {} virtual void _init() { - tm = get_transaction_manager(*segment_manager); + perf_service->add_to_collection(); + tm = get_transaction_manager(*segment_manager, perf_service->get_counters()); segment_cleaner = tm->get_segment_cleaner(); lba_manager = tm->get_lba_manager(); } @@ -123,6 +129,7 @@ protected: } virtual seastar::future<> _teardown() { + perf_service->remove_from_collection(); return tm->close( ).handle_error( crimson::ct_error::assert_all{"Error in teardown"}