From: Yingxin Cheng Date: Wed, 17 Aug 2022 07:06:19 +0000 (+0800) Subject: crimson/os/seastore: move AsyncCleaner to EPM X-Git-Tag: v18.0.0~185^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=5131fcdd68d19d0767d5ae347637428ed4274024;p=ceph-ci.git crimson/os/seastore: move AsyncCleaner to EPM There are two purposes related to generic device tiering: * Make AsyncCleaner (need to introduce a generic interface class later) to hide differences between segment-based tier and RB-based tier cleaner implementations. * Make EPM to coordinate cleaning transactions across tiers with general AsyncCleaners. Signed-off-by: Yingxin Cheng --- diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index c119260e45e..beab454ee9d 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1415,8 +1415,7 @@ void Cache::backref_batch_update( void Cache::complete_commit( Transaction &t, paddr_t final_block_start, - journal_seq_t start_seq, - AsyncCleaner *cleaner) + journal_seq_t start_seq) { LOG_PREFIX(Cache::complete_commit); SUBTRACET(seastore_t, "final_block_start={}, start_seq={}", @@ -1438,11 +1437,7 @@ void Cache::complete_commit( t, is_inline, *i); const auto t_src = t.get_src(); add_extent(i, &t_src); - if (cleaner) { - cleaner->mark_space_used( - i->get_paddr(), - i->get_length()); - } + epm.mark_space_used(i->get_paddr(), i->get_length()); if (is_backref_mapped_extent_node(i)) { DEBUGT("backref_list new {} len {}", t, @@ -1487,18 +1482,12 @@ void Cache::complete_commit( } } - if (cleaner) { - for (auto &i: t.retired_set) { - cleaner->mark_space_free( - i->get_paddr(), - i->get_length()); - } - for (auto &i: t.existing_block_list) { - if (i->is_valid()) { - cleaner->mark_space_used( - i->get_paddr(), - i->get_length()); - } + for (auto &i: t.retired_set) { + epm.mark_space_free(i->get_paddr(), i->get_length()); + } + for (auto &i: t.existing_block_list) { + if (i->is_valid()) { + epm.mark_space_used(i->get_paddr(), i->get_length()); } } diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 4a73058736c..54735cd4d8d 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -26,7 +26,6 @@ class BtreeBackrefManager; namespace crimson::os::seastore { class BackrefManager; -class AsyncCleaner; class SegmentProvider; struct backref_entry_t { @@ -712,8 +711,7 @@ public: void complete_commit( Transaction &t, ///< [in, out] current transaction paddr_t final_block_start, ///< [in] offset of initial block - journal_seq_t seq, ///< [in] journal commit seq - AsyncCleaner *cleaner=nullptr ///< [out] optional segment stat listener + journal_seq_t seq ///< [in] journal commit seq ); /** diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 9518fdc7897..8e4e6dd8137 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -173,4 +173,47 @@ SegmentedOolWriter::alloc_write_ool_extents( }); } +void ExtentPlacementManager::set_async_cleaner(AsyncCleanerRef &&_cleaner) +{ + cleaner = std::move(_cleaner); + writer_refs.clear(); + + ceph_assert(RECLAIM_GENERATIONS > 0); + data_writers_by_gen.resize(RECLAIM_GENERATIONS, {}); + for (reclaim_gen_t gen = 0; gen < RECLAIM_GENERATIONS; ++gen) { + writer_refs.emplace_back(std::make_unique( + data_category_t::DATA, gen, *cleaner, + cleaner->get_ool_segment_seq_allocator())); + data_writers_by_gen[gen] = writer_refs.back().get(); + } + + md_writers_by_gen.resize(RECLAIM_GENERATIONS - 1, {}); + for (reclaim_gen_t gen = 1; gen < RECLAIM_GENERATIONS; ++gen) { + writer_refs.emplace_back(std::make_unique( + data_category_t::METADATA, gen, *cleaner, + cleaner->get_ool_segment_seq_allocator())); + md_writers_by_gen[gen - 1] = writer_refs.back().get(); + } + + for (auto *device : cleaner->get_segment_manager_group() + ->get_segment_managers()) { + add_device(device); + } +} + +void ExtentPlacementManager::set_primary_device(Device *device) +{ + ceph_assert(primary_device == nullptr); + primary_device = device; + if (device->get_device_type() == device_type_t::SEGMENTED) { + prefer_ool = false; + ceph_assert(devices_by_id[device->get_device_id()] == device); + } else { + // RBM device is not in the cleaner. + ceph_assert(device->get_device_type() == device_type_t::RANDOM_BLOCK); + prefer_ool = true; + add_device(primary_device); + } +} + } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 61ed07d0348..d5b39854c15 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -5,6 +5,7 @@ #include "seastar/core/gate.hh" +#include "crimson/os/seastore/async_cleaner.h" #include "crimson/os/seastore/cached_extent.h" #include "crimson/os/seastore/journal/segment_allocator.h" #include "crimson/os/seastore/logging.h" @@ -38,8 +39,6 @@ public: }; using ExtentOolWriterRef = std::unique_ptr; -class SegmentProvider; - /** * SegmentedOolWriter * @@ -90,41 +89,13 @@ public: devices_by_id.resize(DEVICE_ID_MAX, nullptr); } - void init_ool_writers(SegmentProvider &sp, SegmentSeqAllocator &ssa) { - // Currently only one SegmentProvider is supported - writer_refs.clear(); - - ceph_assert(RECLAIM_GENERATIONS > 0); - data_writers_by_gen.resize(RECLAIM_GENERATIONS, {}); - for (reclaim_gen_t gen = 0; gen < RECLAIM_GENERATIONS; ++gen) { - writer_refs.emplace_back(std::make_unique( - data_category_t::DATA, gen, sp, ssa)); - data_writers_by_gen[gen] = writer_refs.back().get(); - } + // TODO: device tiering + void set_async_cleaner(AsyncCleanerRef &&_cleaner); - md_writers_by_gen.resize(RECLAIM_GENERATIONS - 1, {}); - for (reclaim_gen_t gen = 1; gen < RECLAIM_GENERATIONS; ++gen) { - writer_refs.emplace_back(std::make_unique( - data_category_t::METADATA, gen, sp, ssa)); - md_writers_by_gen[gen - 1] = writer_refs.back().get(); - } - } + void set_primary_device(Device *device); - void add_device(Device* device, bool is_primary) { - auto device_id = device->get_device_id(); - ceph_assert(devices_by_id[device_id] == nullptr); - devices_by_id[device_id] = device; - ++num_devices; - if (is_primary) { - ceph_assert(primary_device == nullptr); - primary_device = device; - if (device->get_device_type() == device_type_t::SEGMENTED) { - prefer_ool = false; - } else { - ceph_assert(device->get_device_type() == device_type_t::RANDOM_BLOCK); - prefer_ool = true; - } - } + void set_extent_callback(AsyncCleaner::ExtentCallbackInterface *cb) { + cleaner->set_extent_callback(cb); } seastore_off_t get_block_size() const { @@ -138,8 +109,17 @@ public: return *primary_device; } + store_statfs_t get_stat() const { + return cleaner->stat(); + } + + using mount_ret = AsyncCleaner::mount_ret; + mount_ret mount() { + return cleaner->mount(); + } + using open_ertr = ExtentOolWriter::open_ertr; - open_ertr::future<> open() { + open_ertr::future<> open_for_write() { LOG_PREFIX(ExtentPlacementManager::open); SUBINFO(seastore_journal, "started with {} devices", num_devices); ceph_assert(primary_device != nullptr); @@ -152,6 +132,14 @@ public: }); } + void start_scan_space() { + return cleaner->start_scan_space(); + } + + void start_gc() { + return cleaner->start_gc(); + } + struct alloc_result_t { paddr_t paddr; bufferptr bp; @@ -221,6 +209,7 @@ public: LOG_PREFIX(ExtentPlacementManager::delayed_alloc_or_ool_write); SUBDEBUGT(seastore_journal, "start with {} delayed extents", t, delayed_extents.size()); + assert(writer_refs.size()); return seastar::do_with( std::map>(), [this, &t, &delayed_extents](auto& alloc_map) { @@ -240,6 +229,10 @@ public: }); } + seastar::future<> stop_gc() { + return cleaner->stop(); + } + using close_ertr = ExtentOolWriter::close_ertr; close_ertr::future<> close() { LOG_PREFIX(ExtentPlacementManager::close); @@ -263,7 +256,52 @@ public: return devices_by_id[addr.get_device_id()]->read(addr, len, out); } + void mark_space_used(paddr_t addr, extent_len_t len) { + // TODO: improve tests to drop the cleaner check + if (cleaner) { + cleaner->mark_space_used(addr, len); + } + } + + void mark_space_free(paddr_t addr, extent_len_t len) { + // TODO: improve tests to drop the cleaner check + if (cleaner) { + cleaner->mark_space_free(addr, len); + } + } + + seastar::future<> reserve_projected_usage(std::size_t projected_usage) { + return cleaner->reserve_projected_usage(projected_usage); + } + + void release_projected_usage(std::size_t projected_usage) { + return cleaner->release_projected_usage(projected_usage); + } + + // Testing interfaces + + void test_init_no_background(Device *test_device) { + assert(test_device->get_device_type() == device_type_t::SEGMENTED); + add_device(test_device); + set_primary_device(test_device); + } + + bool check_usage() { + return cleaner->check_usage(); + } + + seastar::future<> run_background_work_until_halt() { + return cleaner->run_until_halt(); + } + private: + void add_device(Device *device) { + auto device_id = device->get_device_id(); + ceph_assert(devices_by_id[device_id] == nullptr); + devices_by_id[device_id] = device; + ++num_devices; + } + ExtentOolWriter* get_writer(placement_hint_t hint, data_category_t category, reclaim_gen_t gen) { @@ -288,7 +326,11 @@ private: std::vector devices_by_id; Device* primary_device = nullptr; std::size_t num_devices = 0; + + // TODO: device tiering + AsyncCleanerRef cleaner; }; + using ExtentPlacementManagerRef = std::unique_ptr; } diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index add1ac2199b..da95f5351a4 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -24,20 +24,18 @@ SET_SUBSYS(seastore_tm); namespace crimson::os::seastore { TransactionManager::TransactionManager( - AsyncCleanerRef _async_cleaner, JournalRef _journal, CacheRef _cache, LBAManagerRef _lba_manager, - ExtentPlacementManagerRef &&epm, - BackrefManagerRef&& backref_manager) - : async_cleaner(std::move(_async_cleaner)), - cache(std::move(_cache)), + ExtentPlacementManagerRef &&_epm, + BackrefManagerRef&& _backref_manager) + : cache(std::move(_cache)), lba_manager(std::move(_lba_manager)), journal(std::move(_journal)), - epm(std::move(epm)), - backref_manager(std::move(backref_manager)) + epm(std::move(_epm)), + backref_manager(std::move(_backref_manager)) { - async_cleaner->set_extent_callback(this); + epm->set_extent_callback(this); journal->set_write_pipeline(&write_pipeline); } @@ -45,13 +43,13 @@ TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs() { LOG_PREFIX(TransactionManager::mkfs); INFO("enter"); - return async_cleaner->mount( + return epm->mount( ).safe_then([this] { return journal->open_for_mkfs(); }).safe_then([this](auto start_seq) { journal->get_trimmer().update_journal_tails(start_seq, start_seq); journal->get_trimmer().set_journal_head(start_seq); - return epm->open(); + return epm->open_for_write(); }).safe_then([this, FNAME]() { return with_transaction_intr( Transaction::src_t::MUTATE, @@ -87,7 +85,7 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount() LOG_PREFIX(TransactionManager::mount); INFO("enter"); cache->init(); - return async_cleaner->mount( + return epm->mount( ).safe_then([this] { return journal->replay( [this]( @@ -121,7 +119,7 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount() return lba_manager->init_cached_extent(t, e); } }).si_then([this, &t] { - async_cleaner->start_scan_space(); + epm->start_scan_space(); return backref_manager->scan_mapped_space( t, [this]( @@ -133,21 +131,21 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount() assert(laddr == L_ADDR_NULL); backref_manager->cache_new_backref_extent(paddr, type); cache->update_tree_extents_num(type, 1); - async_cleaner->mark_space_used(paddr, len); + epm->mark_space_used(paddr, len); } else if (laddr == L_ADDR_NULL) { cache->update_tree_extents_num(type, -1); - async_cleaner->mark_space_free(paddr, len); + epm->mark_space_free(paddr, len); } else { cache->update_tree_extents_num(type, 1); - async_cleaner->mark_space_used(paddr, len); + epm->mark_space_used(paddr, len); } }); }); }); }).safe_then([this] { - return epm->open(); + return epm->open_for_write(); }).safe_then([FNAME, this] { - async_cleaner->start_gc(); + epm->start_gc(); INFO("completed"); }).handle_error( mount_ertr::pass_further{}, @@ -161,7 +159,7 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount() TransactionManager::close_ertr::future<> TransactionManager::close() { LOG_PREFIX(TransactionManager::close); INFO("enter"); - return async_cleaner->stop( + return epm->stop_gc( ).then([this] { return cache->close(); }).safe_then([this] { @@ -282,12 +280,12 @@ TransactionManager::submit_transaction( size_t projected_usage = t.get_allocation_size(); SUBTRACET(seastore_t, "waiting for projected_usage: {}", t, projected_usage); return trans_intr::make_interruptible( - async_cleaner->reserve_projected_usage(projected_usage) + epm->reserve_projected_usage(projected_usage) ).then_interruptible([this, &t] { return submit_transaction_direct(t); }).finally([this, FNAME, projected_usage, &t] { SUBTRACET(seastore_t, "releasing projected_usage: {}", t, projected_usage); - async_cleaner->release_projected_usage(projected_usage); + epm->release_projected_usage(projected_usage); }); }); } @@ -350,8 +348,7 @@ TransactionManager::submit_transaction_direct( cache->complete_commit( tref, submit_result.record_block_base, - start_seq, - async_cleaner.get()); + start_seq); std::vector lba_to_clear; std::vector backref_to_clear; @@ -633,12 +630,10 @@ TransactionManagerRef make_transaction_manager( auto sms = std::make_unique(); auto backref_manager = create_backref_manager(*cache); - epm->add_device(primary_device, true); if (primary_device->get_device_type() == device_type_t::SEGMENTED) { sms->add_segment_manager(static_cast(primary_device)); } for (auto &p_dev : secondary_devices) { - epm->add_device(p_dev, false); ceph_assert(p_dev->get_device_type() == device_type_t::SEGMENTED); sms->add_segment_manager(static_cast(p_dev)); } @@ -676,12 +671,11 @@ TransactionManagerRef make_transaction_manager( ERROR("disabling journal trimming since support for CircularBoundedJournal " "hasn't been added yet"); } - epm->init_ool_writers( - *async_cleaner, - async_cleaner->get_ool_segment_seq_allocator()); + + epm->set_async_cleaner(std::move(async_cleaner)); + epm->set_primary_device(primary_device); return std::make_unique( - std::move(async_cleaner), std::move(journal), std::move(cache), std::move(lba_manager), diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index f4ba48d5ea9..61fe024788d 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -21,7 +21,6 @@ #include "crimson/osd/exceptions.h" #include "crimson/os/seastore/logging.h" -#include "crimson/os/seastore/async_cleaner.h" #include "crimson/os/seastore/seastore_types.h" #include "crimson/os/seastore/cache.h" #include "crimson/os/seastore/lba_manager.h" @@ -65,7 +64,6 @@ public: using base_iertr = Cache::base_iertr; TransactionManager( - AsyncCleanerRef async_cleaner, JournalRef journal, CacheRef cache, LBAManagerRef lba_manager, @@ -620,7 +618,7 @@ public: } store_statfs_t store_stat() const { - return async_cleaner->stat(); + return epm->get_stat(); } ~TransactionManager(); @@ -628,7 +626,6 @@ public: private: friend class Transaction; - AsyncCleanerRef async_cleaner; CacheRef cache; LBAManagerRef lba_manager; JournalRef journal; @@ -643,8 +640,8 @@ private: public: // Testing interfaces - auto get_async_cleaner() { - return async_cleaner.get(); + auto get_epm() { + return epm.get(); } auto get_lba_manager() { diff --git a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc index 208d87b16a8..8cac88c499d 100644 --- a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc +++ b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc @@ -1693,7 +1693,7 @@ TEST_F(d_seastore_tm_test_t, 7_tree_insert_erase_eagain) }); }); }).unsafe_get0(); - async_cleaner->run_until_halt().get0(); + epm->run_background_work_until_halt().get0(); // insert logger().warn("start inserting {} kvs ...", kvs.size()); @@ -1713,7 +1713,7 @@ TEST_F(d_seastore_tm_test_t, 7_tree_insert_erase_eagain) }); }); }).unsafe_get0(); - async_cleaner->run_until_halt().get0(); + epm->run_background_work_until_halt().get0(); ++iter; } } @@ -1759,7 +1759,7 @@ TEST_F(d_seastore_tm_test_t, 7_tree_insert_erase_eagain) }); }); }).unsafe_get0(); - async_cleaner->run_until_halt().get0(); + epm->run_background_work_until_halt().get0(); ++iter; } kvs.erase_from_random(kvs.random_begin(), kvs.random_end()); diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index adc3ce504c6..0718ca72295 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -124,14 +124,14 @@ struct btree_test_base : block_size = segment_manager->get_block_size(); next = segment_id_t{segment_manager->get_device_id(), 0}; sms->add_segment_manager(segment_manager.get()); - epm->add_device(segment_manager.get(), true); + epm->test_init_no_background(segment_manager.get()); journal->set_write_pipeline(&pipeline); return journal->open_for_mkfs().discard_result(); }).safe_then([this] { dummy_tail = journal_seq_t{0, paddr_t::make_seg_paddr(segment_id_t(segment_manager->get_device_id(), 0), 0)}; - return epm->open(); + return epm->open_for_write(); }).safe_then([this] { return seastar::do_with( cache->create_transaction( diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 1785b551111..53e5563a417 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -91,7 +91,7 @@ struct cache_test_t : public seastar_test_suite_t { epm.reset(new ExtentPlacementManager()); cache.reset(new Cache(*epm)); current = paddr_t::make_seg_paddr(segment_id_t(segment_manager->get_device_id(), 0), 0); - epm->add_device(segment_manager.get(), true); + epm->test_init_no_background(segment_manager.get()); return seastar::do_with( get_transaction(), [this](auto &ref_t) { diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index 87a389759af..d01f262af87 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -8,7 +8,6 @@ #include "test/crimson/gtest_seastar.h" #include "test/crimson/seastore/transaction_manager_test_state.h" -#include "crimson/os/seastore/async_cleaner.h" #include "crimson/os/seastore/cache.h" #include "crimson/os/seastore/transaction_manager.h" #include "crimson/os/seastore/segment_manager/ephemeral.h" @@ -400,7 +399,7 @@ struct transaction_manager_test_t : } bool check_usage() { - return async_cleaner->check_usage(); + return epm->check_usage(); } void replay() { @@ -576,7 +575,8 @@ struct transaction_manager_test_t : "try_submit_transaction hit invalid error" } ).then([this](auto ret) { - return async_cleaner->run_until_halt().then([ret] { return ret; }); + return epm->run_background_work_until_halt( + ).then([ret] { return ret; }); }).get0(); if (success) { @@ -626,7 +626,7 @@ struct transaction_manager_test_t : }); }); }).safe_then([this]() { - return async_cleaner->run_until_halt(); + return epm->run_background_work_until_halt(); }).handle_error( crimson::ct_error::assert_all{ "Invalid error in SeaStore::list_collections" diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 47d35557eab..842dada24cb 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -6,8 +6,8 @@ #include #include -#include "crimson/os/seastore/async_cleaner.h" #include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/extent_placement_manager.h" #include "crimson/os/seastore/logging.h" #include "crimson/os/seastore/transaction_manager.h" #include "crimson/os/seastore/segment_manager/ephemeral.h" @@ -163,9 +163,8 @@ class TMTestState : public EphemeralTestState { protected: TransactionManagerRef tm; LBAManager *lba_manager; - BackrefManager *backref_manager; Cache* cache; - AsyncCleaner *async_cleaner; + ExtentPlacementManager *epm; uint64_t seq = 0; TMTestState() : EphemeralTestState(1) {} @@ -185,14 +184,13 @@ protected: } else { tm = make_transaction_manager(segment_manager.get(), sec_devices, true); } - async_cleaner = tm->get_async_cleaner(); + epm = tm->get_epm(); lba_manager = tm->get_lba_manager(); - backref_manager = tm->get_backref_manager(); cache = tm->get_cache(); } virtual void _destroy() override { - async_cleaner = nullptr; + epm = nullptr; lba_manager = nullptr; tm.reset(); } @@ -211,9 +209,9 @@ protected: ).handle_error( crimson::ct_error::assert_all{"Error in mount"} ).then([this] { - return async_cleaner->stop(); + return epm->stop_gc(); }).then([this] { - return async_cleaner->run_until_halt(); + return epm->run_background_work_until_halt(); }); } @@ -278,7 +276,7 @@ protected: void submit_transaction(TransactionRef t) { submit_transaction_fut(*t).unsafe_get0(); - async_cleaner->run_until_halt().get0(); + epm->run_background_work_until_halt().get0(); } };