From: Yingxin Cheng Date: Thu, 24 Mar 2022 03:14:23 +0000 (+0800) Subject: crimson/os/seastore: decouple SegmentManager and TransactionManager X-Git-Tag: v18.0.0~1075^2~10 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=bbf9378115021d27dfa039744bfb57da5df5e230;p=ceph.git crimson/os/seastore: decouple SegmentManager and TransactionManager Signed-off-by: Yingxin Cheng --- diff --git a/src/crimson/os/seastore/device.h b/src/crimson/os/seastore/device.h index bc050a0225fc..33a97a6d70c7 100644 --- a/src/crimson/os/seastore/device.h +++ b/src/crimson/os/seastore/device.h @@ -20,6 +20,8 @@ class Device { public: virtual device_id_t get_device_id() const = 0; + virtual seastore_off_t get_block_size() const = 0; + using read_ertr = crimson::errorator< crimson::ct_error::input_output_error, crimson::ct_error::invarg, diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index e7dddf353415..8865aa9201e5 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -175,10 +175,14 @@ public: devices_by_id.resize(DEVICE_ID_MAX, nullptr); } - void add_device(Device* device) { + void add_device(Device* device, bool is_primary) { auto device_id = device->get_device_id(); ceph_assert(devices_by_id[device_id] == nullptr); devices_by_id[device_id] = device; + if (is_primary) { + ceph_assert(primary_device == nullptr); + primary_device = device; + } } void add_allocator(device_type_t type, ExtentAllocatorRef&& allocator) { @@ -189,6 +193,17 @@ public: allocators[type].size()); } + seastore_off_t get_block_size() const { + assert(primary_device != nullptr); + // assume all the devices have the same block size + return primary_device->get_block_size(); + } + + Device& get_primary_device() { + assert(primary_device != nullptr); + return *primary_device; + } + using open_ertr = ExtentOolWriter::open_ertr; open_ertr::future<> open() { LOG_PREFIX(ExtentPlacementManager::open); @@ -280,6 +295,7 @@ public: allocators.clear(); devices_by_id.clear(); devices_by_id.resize(DEVICE_ID_MAX, nullptr); + primary_device = nullptr; }); } @@ -307,6 +323,7 @@ private: std::map> allocators; std::vector devices_by_id; + Device* primary_device = nullptr; }; using ExtentPlacementManagerRef = std::unique_ptr; diff --git a/src/crimson/os/seastore/extent_reader.h b/src/crimson/os/seastore/extent_reader.h index 68202f6a29ab..b32ae81e64b4 100644 --- a/src/crimson/os/seastore/extent_reader.h +++ b/src/crimson/os/seastore/extent_reader.h @@ -15,12 +15,6 @@ class TransactionManager; class ExtentReader { public: - seastore_off_t get_block_size() const { - assert(segment_managers.size()); - // assume all segment managers have the same block size - return segment_managers[0]->get_block_size(); - } - std::vector& get_segment_managers() { return segment_managers; } @@ -84,6 +78,12 @@ public: found_record_handler_t &handler ///< [in] handler for records ); ///< @return used budget + using release_ertr = SegmentManager::release_ertr; + release_ertr::future<> release_segment(segment_id_t id) { + assert(segment_managers[id.device_id()] != nullptr); + return segment_managers[id.device_id()]->release(id); + } + void add_segment_manager(SegmentManager* segment_manager) { ceph_assert(!segment_managers[segment_manager->get_device_id()]); segment_managers[segment_manager->get_device_id()] = segment_manager; diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 424c0ea8ee9f..9be8bcd5f151 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -189,7 +189,7 @@ SeaStore::mount_ertr::future<> SeaStore::mount() { return segment_manager->mount( ).safe_then([this] { - transaction_manager->add_segment_manager(segment_manager.get()); + transaction_manager->add_segment_manager(segment_manager.get(), true); auto sec_devices = segment_manager->get_secondary_devices(); return crimson::do_for_each(sec_devices, [this](auto& device_entry) { device_id_t id = device_entry.first; @@ -203,7 +203,7 @@ SeaStore::mount_ertr::future<> SeaStore::mount() [this, sm=std::move(sm), magic]() mutable { boost::ignore_unused(magic); // avoid clang warning; assert(sm->get_magic() == magic); - transaction_manager->add_segment_manager(sm.get()); + transaction_manager->add_segment_manager(sm.get(), false); secondaries.emplace_back(std::move(sm)); return seastar::now(); }); @@ -334,7 +334,7 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) }).safe_then([this] { return crimson::do_for_each(secondaries, [this](auto& sec_sm) { return sec_sm->mount().safe_then([this, &sec_sm] { - transaction_manager->add_segment_manager(sec_sm.get()); + transaction_manager->add_segment_manager(sec_sm.get(), false); return seastar::now(); }); }); @@ -342,13 +342,13 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) }).safe_then([this] { return segment_manager->mount(); }).safe_then([this] { - transaction_manager->add_segment_manager(segment_manager.get()); + transaction_manager->add_segment_manager(segment_manager.get(), true); return transaction_manager->mkfs(); }).safe_then([this] { for (auto& sec_sm : secondaries) { - transaction_manager->add_segment_manager(sec_sm.get()); + transaction_manager->add_segment_manager(sec_sm.get(), false); } - transaction_manager->add_segment_manager(segment_manager.get()); + transaction_manager->add_segment_manager(segment_manager.get(), true); return transaction_manager->mount(); }).safe_then([this] { return repeat_eagain([this] { @@ -643,7 +643,7 @@ seastar::future SeaStore::stat( struct stat st; auto &olayout = onode.get_layout(); st.st_size = olayout.size; - st.st_blksize = transaction_manager->get_block_size(); + st.st_blksize = segment_manager->get_block_size(); st.st_blocks = (st.st_size + st.st_blksize - 1) / st.st_blksize; st.st_nlink = 1; DEBUGT("cid {}, oid {}, return size {}", t, c->get_cid(), oid, st.st_size); diff --git a/src/crimson/os/seastore/segment_cleaner.cc b/src/crimson/os/seastore/segment_cleaner.cc index ac48a00a769f..f629f7e81810 100644 --- a/src/crimson/os/seastore/segment_cleaner.cc +++ b/src/crimson/os/seastore/segment_cleaner.cc @@ -683,4 +683,21 @@ SegmentCleaner::scan_extents_ret SegmentCleaner::scan_nonfull_segment( return seastar::now(); } +SegmentCleaner::release_ertr::future<> +SegmentCleaner::maybe_release_segment(Transaction &t) +{ + auto to_release = t.get_segment_to_release(); + if (to_release != NULL_SEG_ID) { + LOG_PREFIX(SegmentCleaner::maybe_release_segment); + INFOT("releasing segment {}", t, to_release); + return scanner->release_segment(to_release + ).safe_then([this, to_release] { + stats.segments_released++; + mark_empty(to_release); + }); + } else { + return SegmentManager::release_ertr::now(); + } +} + } diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index afb400c84cd0..e703b19793d6 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -648,16 +648,6 @@ public: laddr_t laddr, seastore_off_t len) = 0; - /** - * release_segment - * - * Release segment. - */ - using release_segment_ertr = SegmentManager::release_ertr; - using release_segment_ret = release_segment_ertr::future<>; - virtual release_segment_ret release_segment( - segment_id_t id) = 0; - /** * submit_transaction_direct * @@ -776,10 +766,8 @@ public: return segments[id].get_type(); } - void mark_segment_released(segment_id_t segment) { - stats.segments_released++; - return mark_empty(segment); - } + using release_ertr = ExtentReader::release_ertr; + release_ertr::future<> maybe_release_segment(Transaction &t); void adjust_segment_util(double old_usage, double new_usage) { assert(stats.segment_util.buckets[std::floor(old_usage * 10)].count > 0); diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h index 644aaa71444a..22a5aeae0568 100644 --- a/src/crimson/os/seastore/segment_manager.h +++ b/src/crimson/os/seastore/segment_manager.h @@ -214,7 +214,6 @@ public: /* Methods for discovering device geometry, segmentid set, etc */ virtual size_t get_size() const = 0; - virtual seastore_off_t get_block_size() const = 0; virtual seastore_off_t get_segment_size() const = 0; virtual device_segment_id_t get_num_segments() const { ceph_assert(get_size() % get_segment_size() == 0); diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index a40f6632b6b2..31d54a77af60 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -6,7 +6,6 @@ #include "crimson/os/seastore/logging.h" #include "crimson/os/seastore/transaction_manager.h" -#include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/journal.h" /* @@ -23,15 +22,13 @@ SET_SUBSYS(seastore_tm); namespace crimson::os::seastore { TransactionManager::TransactionManager( - SegmentManager &_segment_manager, SegmentCleanerRef _segment_cleaner, JournalRef _journal, CacheRef _cache, LBAManagerRef _lba_manager, ExtentPlacementManagerRef&& epm, ExtentReader& scanner) - : segment_manager(_segment_manager), - segment_cleaner(std::move(_segment_cleaner)), + : segment_cleaner(std::move(_segment_cleaner)), cache(std::move(_cache)), lba_manager(std::move(_lba_manager)), journal(std::move(_journal)), @@ -47,7 +44,7 @@ TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs() LOG_PREFIX(TransactionManager::mkfs); INFO("enter"); return segment_cleaner->mount( - segment_manager.get_device_id() + epm->get_primary_device().get_device_id() ).safe_then([this] { return journal->open_for_write(); }).safe_then([this](auto addr) { @@ -87,7 +84,7 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount() INFO("enter"); cache->init(); return segment_cleaner->mount( - segment_manager.get_device_id() + epm->get_primary_device().get_device_id() ).safe_then([this] { return journal->replay( [this](const auto &offsets, const auto &e, auto last_modified) { @@ -340,16 +337,7 @@ TransactionManager::submit_transaction_direct( lba_manager->complete_transaction(tref); segment_cleaner->update_journal_tail_target( cache->get_oldest_dirty_from().value_or(start_seq)); - auto to_release = tref.get_segment_to_release(); - if (to_release != NULL_SEG_ID) { - SUBDEBUGT(seastore_t, "releasing segment {}", tref, to_release); - return segment_manager.release(to_release - ).safe_then([this, to_release] { - segment_cleaner->mark_segment_released(to_release); - }); - } else { - return SegmentManager::release_ertr::now(); - } + return segment_cleaner->maybe_release_segment(tref); }).safe_then([FNAME, &tref] { SUBTRACET(seastore_t, "completed", tref); return tref.get_handle().complete(); @@ -569,7 +557,6 @@ TransactionManagerRef make_transaction_manager( auto lba_manager = lba_manager::create_lba_manager(sm, *cache); return std::make_unique( - sm, std::move(segment_cleaner), std::move(journal), std::move(cache), diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 34171b63f98a..46f78ded4909 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -24,7 +24,6 @@ #include "crimson/os/seastore/segment_cleaner.h" #include "crimson/os/seastore/seastore_types.h" #include "crimson/os/seastore/cache.h" -#include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/lba_manager.h" #include "crimson/os/seastore/journal.h" #include "crimson/os/seastore/extent_placement_manager.h" @@ -64,7 +63,6 @@ public: using base_iertr = Cache::base_iertr; TransactionManager( - SegmentManager &segment_manager, SegmentCleanerRef segment_cleaner, JournalRef journal, CacheRef cache, @@ -409,15 +407,6 @@ public: laddr_t laddr, seastore_off_t len) final; - using release_segment_ret = - SegmentCleaner::ExtentCallbackInterface::release_segment_ret; - release_segment_ret release_segment( - segment_id_t id) final { - LOG_PREFIX(TransactionManager::release_segment); - SUBDEBUG(seastore_tm, "{}", id); - return segment_manager.release(id); - } - /** * read_root_meta * @@ -534,18 +523,19 @@ public: } extent_len_t get_block_size() const { - return segment_manager.get_block_size(); + return epm->get_block_size(); } store_statfs_t store_stat() const { return segment_cleaner->stat(); } - void add_segment_manager(SegmentManager* sm) { + void add_segment_manager(SegmentManager* sm, bool is_primary) { LOG_PREFIX(TransactionManager::add_segment_manager); - SUBDEBUG(seastore_tm, "adding segment manager {}", sm->get_device_id()); + SUBDEBUG(seastore_tm, "adding segment manager {}, is_primary={}", + sm->get_device_id(), is_primary); scanner.add_segment_manager(sm); - epm->add_device(sm); + epm->add_device(sm, is_primary); epm->add_allocator( device_type_t::SEGMENTED, std::make_unique( @@ -559,10 +549,6 @@ public: private: friend class Transaction; - // although there might be multiple devices backing seastore, - // only one of them are supposed to hold the journal. This - // segment manager is that device - SegmentManager &segment_manager; SegmentCleanerRef segment_cleaner; CacheRef cache; LBAManagerRef lba_manager; diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc index 851228e2ad42..54f6912349af 100644 --- a/src/crimson/tools/store_nbd/tm_driver.cc +++ b/src/crimson/tools/store_nbd/tm_driver.cc @@ -135,7 +135,7 @@ seastar::future TMDriver::read( void TMDriver::init() { tm = make_transaction_manager(*segment_manager, false /* detailed */); - tm->add_segment_manager(segment_manager.get()); + tm->add_segment_manager(segment_manager.get(), true); } void TMDriver::clear() diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc index 77578f3625f8..3ff93fb4f85d 100644 --- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc +++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc @@ -84,7 +84,7 @@ struct fltree_onode_manager_test_t virtual FuturizedStore::mkfs_ertr::future<> _mkfs() final { return TMTestState::_mkfs( ).safe_then([this] { - tm->add_segment_manager(segment_manager.get()); + tm->add_segment_manager(segment_manager.get(), true); return tm->mount( ).safe_then([this] { return repeat_eagain([this] { diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 79bbce63c6aa..f0cf4e03dd85 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -108,7 +108,7 @@ struct btree_test_base : block_size = segment_manager->get_block_size(); next = segment_id_t{segment_manager->get_device_id(), 0}; scanner_ref.add_segment_manager(segment_manager.get()); - epm->add_device(segment_manager.get()); + epm->add_device(segment_manager.get(), true); journal->set_write_pipeline(&pipeline); return segment_manager->init( diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 94efc4fd48db..5840739448be 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -86,7 +86,7 @@ struct cache_test_t : public seastar_test_suite_t { epm.reset(new ExtentPlacementManager()); cache.reset(new Cache(*epm)); current = paddr_t::make_seg_paddr(segment_id_t(segment_manager->get_device_id(), 0), 0); - epm->add_device(segment_manager.get()); + epm->add_device(segment_manager.get(), true); return segment_manager->init( ).safe_then([this] { return seastar::do_with( diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 7c0d3df0878d..1dd72325f672 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -93,7 +93,7 @@ protected: virtual void _init() override { tm = make_transaction_manager(*segment_manager, true); - tm->add_segment_manager(segment_manager.get()); + tm->add_segment_manager(segment_manager.get(), true); segment_cleaner = tm->get_segment_cleaner(); lba_manager = tm->get_lba_manager(); }