From: Yingxin Cheng Date: Thu, 27 Jan 2022 15:35:45 +0000 (+0800) Subject: crimson/os/seastore: cleanup dependency between epm and lba_manager X-Git-Tag: v18.0.0~1401^2~4 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=d83627c066003a18008d535c6fac346785154d50;p=ceph.git crimson/os/seastore: cleanup dependency between epm and lba_manager Move lba_manamger->update_mapping() from epm to transaction manager. Signed-off-by: Yingxin Cheng --- diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 1270881241530..5a711a1b45c8f 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -25,8 +25,10 @@ SET_SUBSYS(seastore_cache); namespace crimson::os::seastore { Cache::Cache( - ExtentReader &reader) + ExtentReader &reader, + ExtentPlacementManager &epm) : reader(reader), + epm(epm), lru(crimson::common::get_conf( "seastore_cache_lru_size")) { diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 524b72707ec46..2d7042bd643a3 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -100,13 +100,9 @@ public: crimson::ct_error::input_output_error>; using base_iertr = trans_iertr; - Cache(ExtentReader &reader); + Cache(ExtentReader &reader, ExtentPlacementManager &epm); ~Cache(); - void set_epm(ExtentPlacementManager& epm) { - p_epm = &epm; - } - /// Creates empty transaction by source TransactionRef create_transaction( Transaction::src_t src, @@ -501,7 +497,7 @@ public: LOG_PREFIX(Cache::alloc_new_extent); SUBTRACET(seastore_cache, "allocate {} {}B, hint={}", t, T::TYPE, length, hint); - auto result = p_epm->alloc_new_extent(t, T::TYPE, length, hint); + auto result = epm.alloc_new_extent(t, T::TYPE, length, hint); auto ret = CachedExtent::make_cached_extent_ref(std::move(result.bp)); ret->set_paddr(result.paddr); t.add_fresh_extent(ret); @@ -740,7 +736,7 @@ public: private: ExtentReader &reader; ///< ref to extent reader - ExtentPlacementManager* p_epm = nullptr; + ExtentPlacementManager& epm; RootBlockRef root; ///< ref to current root ExtentIndex extents; ///< set of live extents diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 71d86e00c8bb2..6db0b9b9c7631 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -3,7 +3,6 @@ #include "crimson/os/seastore/extent_placement_manager.h" -#include "crimson/os/seastore/lba_manager.h" #include "crimson/os/seastore/segment_cleaner.h" namespace { @@ -19,11 +18,9 @@ namespace crimson::os::seastore { SegmentedAllocator::SegmentedAllocator( SegmentProvider& sp, SegmentManager& sm, - LBAManager& lba_manager, Journal& journal) : segment_provider(sp), segment_manager(sm), - lba_manager(lba_manager), journal(journal) { std::generate_n( @@ -34,39 +31,10 @@ SegmentedAllocator::SegmentedAllocator( return Writer{ segment_provider, segment_manager, - lba_manager, journal}; }); } -SegmentedAllocator::Writer::finish_record_ret -SegmentedAllocator::Writer::finish_write( - Transaction& t, - ool_record_t& record) { - return trans_intr::do_for_each(record.get_extents(), - [this, &t](auto& ool_extent) { - LOG_PREFIX(SegmentedAllocator::Writer::finish_write); - auto& lextent = ool_extent.get_lextent(); - DEBUGT("extent: {}, ool_paddr: {}", - t, - *lextent, - ool_extent.get_ool_paddr()); - return lba_manager.update_mapping( - t, - lextent->get_laddr(), - lextent->get_paddr(), - ool_extent.get_ool_paddr() - ).si_then([&ool_extent, &t, &lextent, this, FNAME] { - lextent->hint = {}; - TRACET("mark extent as ool at {} -- {}", t, ool_extent.get_ool_paddr(), *lextent); - t.mark_delayed_extent_ool(lextent, ool_extent.get_ool_paddr()); - return finish_record_iertr::now(); - }); - }).si_then([&record] { - record.clear(); - }); -} - SegmentedAllocator::Writer::write_iertr::future<> SegmentedAllocator::Writer::_write( Transaction& t, @@ -104,10 +72,9 @@ SegmentedAllocator::Writer::_write( return trans_intr::make_interruptible( current_segment->segment->write(record.get_base(), bl).safe_then( - [this, pr=std::move(pr), &t, - it=(--current_segment->inflight_writes.end()), - cs=current_segment]() mutable { - LOG_PREFIX(SegmentedAllocator::Writer::_write); + [this, FNAME, pr=std::move(pr), &t, + it=(--current_segment->inflight_writes.end()), + cs=current_segment]() mutable { if (cs->outdated) { DEBUGT("segment rolled", t); pr.set_value(); @@ -117,8 +84,15 @@ SegmentedAllocator::Writer::_write( } return seastar::now(); }) - ).si_then([this, &record, &t]() mutable { - return finish_write(t, record); + ).si_then([FNAME, &record, &t] { + for (auto& ool_extent : record.get_extents()) { + auto& lextent = ool_extent.get_lextent(); + auto paddr = ool_extent.get_ool_paddr(); + TRACET("ool extent written at {} -- {}", t, *lextent, paddr); + lextent->hint = {}; + t.mark_delayed_extent_ool(lextent, paddr); + } + record.clear(); }); } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 9d56e3e026353..b78e5b28b2eea 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -163,7 +163,6 @@ struct open_segment_wrapper_t : public boost::intrusive_ref_counter< using open_segment_wrapper_ref = boost::intrusive_ptr; -class LBAManager; class SegmentProvider; /** @@ -185,11 +184,9 @@ class SegmentedAllocator : public ExtentAllocator { Writer( SegmentProvider& sp, SegmentManager& sm, - LBAManager& lba_manager, Journal& journal) : segment_provider(sp), segment_manager(sm), - lba_manager(lba_manager), journal(journal) {} Writer(Writer &&) = default; @@ -205,13 +202,6 @@ class SegmentedAllocator : public ExtentAllocator { }); } private: - using finish_record_ertr = crimson::errorator< - crimson::ct_error::input_output_error>; - using finish_record_iertr = trans_iertr; - using finish_record_ret = finish_record_iertr::future<>; - finish_record_ret finish_write( - Transaction& t, - ool_record_t& record); bool _needs_roll(seastore_off_t length) const; write_iertr::future<> _write( @@ -235,7 +225,6 @@ class SegmentedAllocator : public ExtentAllocator { open_segment_wrapper_ref current_segment; std::list open_segments; seastore_off_t allocated_to = 0; - LBAManager& lba_manager; Journal& journal; crimson::condition_variable segment_rotation_guard; seastar::gate writer_guard; @@ -245,7 +234,6 @@ public: SegmentedAllocator( SegmentProvider& sp, SegmentManager& sm, - LBAManager& lba_manager, Journal& journal); Writer &get_writer(placement_hint_t hint) { @@ -281,15 +269,12 @@ private: SegmentProvider& segment_provider; SegmentManager& segment_manager; std::vector writers; - LBAManager& lba_manager; Journal& journal; }; class ExtentPlacementManager { public: - ExtentPlacementManager( - LBAManager& lba_manager - ) : lba_manager(lba_manager) {} + ExtentPlacementManager() = default; struct alloc_result_t { paddr_t paddr; @@ -332,34 +317,25 @@ public: /** * delayed_alloc_or_ool_write * - * Performs any outstanding ool writes and updates pending lba updates - * accordingly + * Performs delayed allocation and do writes for out-of-line extents. */ using alloc_paddr_iertr = ExtentOolWriter::write_iertr; alloc_paddr_iertr::future<> delayed_alloc_or_ool_write( - Transaction& t) { + Transaction& t, + const std::list& delayed_extents) { LOG_PREFIX(ExtentPlacementManager::delayed_alloc_or_ool_write); - SUBDEBUGT(seastore_tm, "start", t); + SUBDEBUGT(seastore_tm, "start with {} delayed extents", + t, delayed_extents.size()); return seastar::do_with( std::map>(), - [this, &t](auto& alloc_map) { - LOG_PREFIX(ExtentPlacementManager::delayed_alloc_or_ool_write); - auto& alloc_list = t.get_delayed_alloc_list(); - uint64_t num_ool_extents = 0; - for (auto& extent : alloc_list) { - // extents may be invalidated - if (!extent->is_valid()) { - t.increment_delayed_invalid_extents(); - continue; - } + [this, &t, &delayed_extents](auto& alloc_map) { + for (auto& extent : delayed_extents) { // For now, just do ool allocation for any delayed extent auto& allocator_ptr = get_allocator( get_allocator_type(extent->hint), extent->hint ); alloc_map[allocator_ptr.get()].emplace_back(extent); - num_ool_extents++; } - SUBDEBUGT(seastore_tm, "{} ool extents", t, num_ool_extents); return trans_intr::do_for_each(alloc_map, [&t](auto& p) { auto allocator = p.first; auto& extents = p.second; @@ -388,7 +364,6 @@ private: return devices[std::rand() % devices.size()]; } - LBAManager& lba_manager; std::map> allocators; }; using ExtentPlacementManagerRef = std::unique_ptr; diff --git a/src/crimson/os/seastore/lba_manager.cc b/src/crimson/os/seastore/lba_manager.cc index 73411dcf7e3ac..c6026ad246224 100644 --- a/src/crimson/os/seastore/lba_manager.cc +++ b/src/crimson/os/seastore/lba_manager.cc @@ -6,9 +6,43 @@ #include "crimson/os/seastore/lba_manager.h" #include "crimson/os/seastore/lba_manager/btree/btree_lba_manager.h" -namespace crimson::os::seastore::lba_manager { +namespace crimson::os::seastore { -LBAManagerRef create_lba_manager( +LBAManager::update_mappings_ret +LBAManager::update_mappings( + Transaction& t, + const std::list& extents, + const std::vector& original_paddrs) +{ + assert(extents.size() == original_paddrs.size()); + auto extents_end = extents.end(); + return seastar::do_with( + extents.begin(), + original_paddrs.begin(), + [this, extents_end, &t](auto& iter_extents, + auto& iter_original_paddrs) { + return trans_intr::repeat( + [this, extents_end, &t, &iter_extents, &iter_original_paddrs] + { + if (extents_end == iter_extents) { + return update_mappings_iertr::make_ready_future< + seastar::stop_iteration>(seastar::stop_iteration::yes); + } + return update_mapping( + t, + (*iter_extents)->get_laddr(), + *iter_original_paddrs, + (*iter_extents)->get_paddr() + ).si_then([&iter_extents, &iter_original_paddrs] { + ++iter_extents; + ++iter_original_paddrs; + return seastar::stop_iteration::no; + }); + }); + }); +} + +LBAManagerRef lba_manager::create_lba_manager( SegmentManager &segment_manager, Cache &cache) { return LBAManagerRef(new btree::BtreeLBAManager(segment_manager, cache)); diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h index fe0a5fa331bcb..172052bac55ba 100644 --- a/src/crimson/os/seastore/lba_manager.h +++ b/src/crimson/os/seastore/lba_manager.h @@ -175,6 +175,19 @@ public: laddr_t laddr, paddr_t prev_addr, paddr_t paddr) = 0; + + /** + * update_mappings + * + * update lba mappings for delayed allocated extents + */ + using update_mappings_iertr = update_mapping_iertr; + using update_mappings_ret = update_mapping_ret; + update_mappings_ret update_mappings( + Transaction& t, + const std::list& extents, + const std::vector& original_paddrs); + /** * get_physical_extent_if_live * diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index bc3e5ca7674fd..1ff54fca9fd77 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -1415,12 +1415,10 @@ seastar::future> make_seastore( false /* detailed */); auto journal = std::make_unique(*sm, scanner_ref); - auto cache = std::make_unique(scanner_ref); + auto epm = std::make_unique(); + auto cache = std::make_unique(scanner_ref, *epm); auto lba_manager = lba_manager::create_lba_manager(*sm, *cache); - auto epm = std::make_unique(*lba_manager); - cache->set_epm(*epm); - journal->set_segment_provider(&*segment_cleaner); auto tm = std::make_unique( diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index 4090744412841..d613261fa366a 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -190,8 +190,18 @@ public: return to_release; } - auto& get_delayed_alloc_list() { - return delayed_alloc_list; + auto get_delayed_alloc_list() { + std::list ret; + for (auto& extent : delayed_alloc_list) { + // delayed extents may be invalidated + if (extent->is_valid()) { + ret.push_back(std::move(extent)); + } else { + ++num_delayed_invalid_extents; + } + } + delayed_alloc_list.clear(); + return ret; } const auto &get_mutated_block_list() { @@ -348,10 +358,6 @@ public: return ool_write_stats; } - void increment_delayed_invalid_extents() { - ++num_delayed_invalid_extents; - } - private: friend class Cache; friend Ref make_test_transaction(); diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index db37abdd09634..ec80eb9234177 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -285,12 +285,29 @@ TransactionManager::submit_transaction_direct( return trans_intr::make_interruptible( tref.get_handle().enter(write_pipeline.ool_writes) ).then_interruptible([this, FNAME, &tref] { - SUBTRACET(seastore_t, "process delayed and out-of-line extents", tref); - return epm->delayed_alloc_or_ool_write(tref - ).handle_error_interruptible( - crimson::ct_error::input_output_error::pass_further(), - crimson::ct_error::assert_all("invalid error") - ); + auto delayed_extents = tref.get_delayed_alloc_list(); + auto num_extents = delayed_extents.size(); + SUBTRACET(seastore_t, "process {} delayed extents", tref, num_extents); + std::vector delayed_paddrs; + delayed_paddrs.reserve(num_extents); + for (auto& ext : delayed_extents) { + assert(ext->get_paddr().is_delayed()); + delayed_paddrs.push_back(ext->get_paddr()); + } + return seastar::do_with( + std::move(delayed_extents), + std::move(delayed_paddrs), + [this, FNAME, &tref](auto& delayed_extents, auto& delayed_paddrs) + { + return epm->delayed_alloc_or_ool_write(tref, delayed_extents + ).si_then([this, FNAME, &tref, &delayed_extents, &delayed_paddrs] { + SUBTRACET(seastore_t, "update delayed extent mappings", tref); + return lba_manager->update_mappings(tref, delayed_extents, delayed_paddrs); + }).handle_error_interruptible( + crimson::ct_error::input_output_error::pass_further(), + crimson::ct_error::assert_all("invalid error") + ); + }); }).si_then([this, FNAME, &tref] { SUBTRACET(seastore_t, "about to prepare", tref); return tref.get_handle().enter(write_pipeline.prepare); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index a5bb254badb8a..a7371ae4a3ce6 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -550,7 +550,6 @@ public: std::make_unique( *segment_cleaner, *sm, - *lba_manager, *journal)); } diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc index 19cbd56e5e629..3ac4cccef1616 100644 --- a/src/crimson/tools/store_nbd/tm_driver.cc +++ b/src/crimson/tools/store_nbd/tm_driver.cc @@ -140,20 +140,17 @@ void TMDriver::init() false /* detailed */); std::vector sms; segment_cleaner->mount(segment_manager->get_device_id(), sms); - auto journal = std::make_unique(*segment_manager, scanner_ref); - auto cache = std::make_unique(scanner_ref); + auto journal = std::make_unique(*segment_manager, *scanner); + auto epm = std::make_unique(); + auto cache = std::make_unique(scanner_ref, *epm); auto lba_manager = lba_manager::create_lba_manager(*segment_manager, *cache); - auto epm = std::make_unique(*cache, *lba_manager); - epm->add_allocator( device_type_t::SEGMENTED, std::make_unique( *segment_cleaner, *segment_manager, - *lba_manager, - *journal, - *cache)); + *journal)); journal->set_segment_provider(&*segment_cleaner); diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index f9d5d195057b1..7bf2e01a4ad38 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -30,6 +30,7 @@ struct btree_test_base : segment_manager::EphemeralSegmentManagerRef segment_manager; ExtentReaderRef scanner; JournalRef journal; + ExtentPlacementManagerRef epm; CacheRef cache; size_t block_size; @@ -74,7 +75,8 @@ struct btree_test_base : segment_manager = segment_manager::create_test_ephemeral(); scanner.reset(new ExtentReader()); journal.reset(new Journal(*segment_manager, *scanner)); - cache.reset(new Cache(*scanner)); + epm.reset(new ExtentPlacementManager()); + cache.reset(new Cache(*scanner, *epm)); block_size = segment_manager->get_block_size(); next = segment_id_t{segment_manager->get_device_id(), 0}; @@ -117,6 +119,7 @@ struct btree_test_base : segment_manager.reset(); scanner.reset(); journal.reset(); + epm.reset(); cache.reset(); }).handle_error( crimson::ct_error::all_same_way([] { diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 4b49f020acf5a..90bf0f4d485b6 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -22,6 +22,7 @@ namespace { struct cache_test_t : public seastar_test_suite_t { segment_manager::EphemeralSegmentManagerRef segment_manager; ExtentReaderRef reader; + ExtentPlacementManagerRef epm; CacheRef cache; paddr_t current; journal_seq_t seq; @@ -84,7 +85,8 @@ struct cache_test_t : public seastar_test_suite_t { seastar::future<> set_up_fut() final { segment_manager = segment_manager::create_test_ephemeral(); reader.reset(new ExtentReader()); - cache.reset(new Cache(*reader)); + epm.reset(new ExtentPlacementManager()); + cache.reset(new Cache(*reader, *epm)); current = paddr_t::make_seg_paddr(segment_id_t(segment_manager->get_device_id(), 0), 0); reader->add_segment_manager(segment_manager.get()); return segment_manager->init( @@ -112,6 +114,7 @@ struct cache_test_t : public seastar_test_suite_t { ).safe_then([this] { segment_manager.reset(); reader.reset(); + epm.reset(); cache.reset(); }).handle_error( Cache::close_ertr::assert_all{} diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 9213d2b8243a0..c177eafbf1011 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -80,19 +80,16 @@ auto get_transaction_manager( std::move(scanner), true); auto journal = std::make_unique(segment_manager, scanner_ref); - auto cache = std::make_unique(scanner_ref); + auto epm = std::make_unique(); + auto cache = std::make_unique(scanner_ref, *epm); auto lba_manager = lba_manager::create_lba_manager(segment_manager, *cache); - auto epm = std::make_unique(*cache, *lba_manager); - epm->add_allocator( device_type_t::SEGMENTED, std::make_unique( *segment_cleaner, segment_manager, - *lba_manager, - *journal, - *cache)); + *journal)); journal->set_segment_provider(&*segment_cleaner);