From 3c28640ced5a05332da9ec80ffdbf5c3c75da3a0 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Mon, 5 Sep 2022 13:54:22 +0900 Subject: [PATCH] crimson/os/seastore/rbm: add initialization step for rbm ool This commit adds init step for rbm ool before starting unittest-transaction-manager Signed-off-by: Myoungwon Oh --- src/crimson/os/seastore/async_cleaner.cc | 18 ++++- src/crimson/os/seastore/async_cleaner.h | 21 ++++- .../os/seastore/extent_placement_manager.cc | 81 ++++++++++++------- .../os/seastore/extent_placement_manager.h | 3 + .../os/seastore/random_block_manager.h | 2 + .../random_block_manager/block_rb_manager.h | 6 ++ .../random_block_manager/rbm_device.h | 1 + .../os/seastore/randomblock_manager_group.h | 71 ++++++++++++++++ .../os/seastore/transaction_manager.cc | 27 ++++--- .../seastore/test_transaction_manager.cc | 5 ++ .../seastore/transaction_manager_test_state.h | 43 ++++++---- 11 files changed, 223 insertions(+), 55 deletions(-) create mode 100644 src/crimson/os/seastore/randomblock_manager_group.h diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc index 943f3303b19..84d5260a1ff 100644 --- a/src/crimson/os/seastore/async_cleaner.cc +++ b/src/crimson/os/seastore/async_cleaner.cc @@ -1543,9 +1543,11 @@ void SegmentCleaner::print(std::ostream &os, bool is_detailed) const } RBMCleaner::RBMCleaner( + RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, bool detailed) : detailed(detailed), + rb_group(std::move(rb_group)), backref_manager(backref_manager) {} @@ -1594,7 +1596,21 @@ RBMCleaner::clean_space_ret RBMCleaner::clean_space() RBMCleaner::mount_ret RBMCleaner::mount() { stats = {}; - return mount_ertr::now(); + return seastar::do_with( + rb_group->get_rb_managers(), + [](auto &rbs) { + return crimson::do_for_each( + rbs.begin(), + rbs.end(), + [](auto& it) { + return it->open( + ).handle_error( + crimson::ct_error::input_output_error::pass_further(), + crimson::ct_error::assert_all{ + "Invalid error when opening RBM"} + ); + }); + }); } } diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 81b38df2b45..3f5a312f292 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -14,6 +14,7 @@ #include "crimson/os/seastore/seastore_types.h" #include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/segment_manager_group.h" +#include "crimson/os/seastore/randomblock_manager_group.h" #include "crimson/os/seastore/transaction.h" #include "crimson/os/seastore/segment_seq_allocator.h" @@ -1290,14 +1291,20 @@ using RBMCleanerRef = std::unique_ptr; class RBMCleaner : public AsyncCleaner { public: RBMCleaner( + RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, bool detailed); static RBMCleanerRef create( + RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, bool detailed) { return std::make_unique( - backref_manager, detailed); + std::move(rb_group), backref_manager, detailed); + } + + RBMDeviceGroup* get_rb_group() { + return rb_group.get(); } /* @@ -1340,6 +1347,16 @@ public: clean_space_ret clean_space() final; + RandomBlockManager* get_rbm(paddr_t paddr) { + auto rbs = rb_group->get_rb_managers(); + for (auto p : rbs) { + if (p->get_device_id() == paddr.get_device_id()) { + return p; + } + } + return nullptr; + } + // Testing interfaces bool check_usage() final { @@ -1354,7 +1371,7 @@ public: private: const bool detailed; - + RBMDeviceGroupRef rb_group; BackrefManager &backref_manager; diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index e6ddaf0a94d..500a2bed66a 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -178,28 +178,45 @@ void ExtentPlacementManager::init( { writer_refs.clear(); - auto segment_cleaner = dynamic_cast(cleaner.get()); - ceph_assert(segment_cleaner != nullptr); - auto num_writers = generation_to_writer(REWRITE_GENERATIONS); - data_writers_by_gen.resize(num_writers, {}); - for (rewrite_gen_t gen = OOL_GENERATION; gen < REWRITE_GENERATIONS; ++gen) { - writer_refs.emplace_back(std::make_unique( - data_category_t::DATA, gen, *segment_cleaner, - segment_cleaner->get_ool_segment_seq_allocator())); - data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); - } + if (trimmer->get_journal_type() == journal_type_t::SEGMENTED) { + auto segment_cleaner = dynamic_cast(cleaner.get()); + ceph_assert(segment_cleaner != nullptr); + auto num_writers = generation_to_writer(REWRITE_GENERATIONS); + data_writers_by_gen.resize(num_writers, {}); + for (rewrite_gen_t gen = OOL_GENERATION; gen < REWRITE_GENERATIONS; ++gen) { + writer_refs.emplace_back(std::make_unique( + data_category_t::DATA, gen, *segment_cleaner, + segment_cleaner->get_ool_segment_seq_allocator())); + data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); + } - md_writers_by_gen.resize(num_writers, {}); - for (rewrite_gen_t gen = OOL_GENERATION; gen < REWRITE_GENERATIONS; ++gen) { - writer_refs.emplace_back(std::make_unique( - data_category_t::METADATA, gen, *segment_cleaner, - segment_cleaner->get_ool_segment_seq_allocator())); - md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); - } + md_writers_by_gen.resize(num_writers, {}); + for (rewrite_gen_t gen = OOL_GENERATION; gen < REWRITE_GENERATIONS; ++gen) { + writer_refs.emplace_back(std::make_unique( + data_category_t::METADATA, gen, *segment_cleaner, + segment_cleaner->get_ool_segment_seq_allocator())); + md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); + } - for (auto *device : segment_cleaner->get_segment_manager_group() - ->get_segment_managers()) { - add_device(device); + for (auto *device : segment_cleaner->get_segment_manager_group() + ->get_segment_managers()) { + add_device(device); + } + } else { + assert(trimmer->get_journal_type() == journal_type_t::RANDOM_BLOCK); + auto rb_cleaner = dynamic_cast(cleaner.get()); + ceph_assert(rb_cleaner != nullptr); + auto num_writers = generation_to_writer(REWRITE_GENERATIONS); + data_writers_by_gen.resize(num_writers, {}); + md_writers_by_gen.resize(num_writers, {}); + writer_refs.emplace_back(std::make_unique( + rb_cleaner)); + // TODO: implement eviction in RBCleaner and introduce further writers + data_writers_by_gen[generation_to_writer(OOL_GENERATION)] = writer_refs.back().get(); + md_writers_by_gen[generation_to_writer(OOL_GENERATION)] = writer_refs.back().get(); + for (auto *rb : rb_cleaner->get_rb_group()->get_rb_managers()) { + add_device(rb->get_device()); + } } background_process.init(std::move(trimmer), std::move(cleaner)); @@ -211,13 +228,11 @@ void ExtentPlacementManager::set_primary_device(Device *device) primary_device = device; if (device->get_backend_type() == backend_type_t::SEGMENTED) { prefer_ool = false; - ceph_assert(devices_by_id[device->get_device_id()] == device); } else { - // RBM device is not in the cleaner. ceph_assert(device->get_backend_type() == backend_type_t::RANDOM_BLOCK); prefer_ool = true; - add_device(primary_device); } + ceph_assert(devices_by_id[device->get_device_id()] == device); } ExtentPlacementManager::open_ertr::future<> @@ -227,10 +242,16 @@ ExtentPlacementManager::open_for_write() INFO("started with {} devices", num_devices); ceph_assert(primary_device != nullptr); return crimson::do_for_each(data_writers_by_gen, [](auto &writer) { - return writer->open(); + if (writer) { + return writer->open(); + } + return open_ertr::now(); }).safe_then([this] { return crimson::do_for_each(md_writers_by_gen, [](auto &writer) { - return writer->open(); + if (writer) { + return writer->open(); + } + return open_ertr::now(); }); }); } @@ -269,10 +290,16 @@ ExtentPlacementManager::close() LOG_PREFIX(ExtentPlacementManager::close); INFO("started"); return crimson::do_for_each(data_writers_by_gen, [](auto &writer) { - return writer->close(); + if (writer) { + return writer->close(); + } + return close_ertr::now(); }).safe_then([this] { return crimson::do_for_each(md_writers_by_gen, [](auto &writer) { - return writer->close(); + if (writer) { + return writer->close(); + } + return close_ertr::now(); }); }); } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 42ae8a89dd2..061aa54a6a8 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -9,6 +9,9 @@ #include "crimson/os/seastore/cached_extent.h" #include "crimson/os/seastore/journal/segment_allocator.h" #include "crimson/os/seastore/transaction.h" +#include "crimson/os/seastore/random_block_manager.h" +#include "crimson/os/seastore/random_block_manager/block_rb_manager.h" +#include "crimson/os/seastore/randomblock_manager_group.h" namespace crimson::os::seastore { diff --git a/src/crimson/os/seastore/random_block_manager.h b/src/crimson/os/seastore/random_block_manager.h index 88d1be3707d..056bc9a1bcf 100644 --- a/src/crimson/os/seastore/random_block_manager.h +++ b/src/crimson/os/seastore/random_block_manager.h @@ -104,6 +104,8 @@ public: virtual extent_len_t get_block_size() const = 0; virtual uint64_t get_free_blocks() const = 0; virtual device_id_t get_device_id() const = 0; + virtual const seastore_meta_t &get_meta() const = 0; + virtual Device* get_device() = 0; virtual ~RandomBlockManager() {} }; using RandomBlockManagerRef = std::unique_ptr; diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h index 4059cbf2270..210de43c59a 100644 --- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h +++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h @@ -85,6 +85,12 @@ public: assert(device); return get_size() / get_block_size(); } + const seastore_meta_t &get_meta() const final { + return device->get_meta(); + } + RBMDevice* get_device() { + return device; + } private: /* diff --git a/src/crimson/os/seastore/random_block_manager/rbm_device.h b/src/crimson/os/seastore/random_block_manager/rbm_device.h index 10a7da0969f..acd8baee446 100644 --- a/src/crimson/os/seastore/random_block_manager/rbm_device.h +++ b/src/crimson/os/seastore/random_block_manager/rbm_device.h @@ -176,6 +176,7 @@ public: super.journal_size = size; } }; +using RBMDeviceRef = std::unique_ptr; class EphemeralRBMDevice : public RBMDevice { public: diff --git a/src/crimson/os/seastore/randomblock_manager_group.h b/src/crimson/os/seastore/randomblock_manager_group.h new file mode 100644 index 00000000000..77d9cf7975c --- /dev/null +++ b/src/crimson/os/seastore/randomblock_manager_group.h @@ -0,0 +1,71 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- +// vim: ts=8 sw=2 smarttab expandtab + +#pragma once + +#include + +#include "crimson/common/errorator.h" +#include "crimson/os/seastore/seastore_types.h" +#include "crimson/os/seastore/random_block_manager.h" +#include "crimson/os/seastore/random_block_manager/block_rb_manager.h" + +namespace crimson::os::seastore { + +class RBMDeviceGroup { +public: + RBMDeviceGroup() { + rb_devices.resize(DEVICE_ID_MAX); + } + + const std::set& get_device_ids() const { + return device_ids; + } + + std::vector get_rb_managers() const { + assert(device_ids.size()); + std::vector ret; + for (auto& device_id : device_ids) { + auto rb_device = rb_devices[device_id].get(); + assert(rb_device->get_device_id() == device_id); + ret.emplace_back(rb_device); + } + return ret; + } + + void add_rb_manager(RandomBlockManagerRef rbm) { + auto device_id = rbm->get_device_id(); + ceph_assert(!has_device(device_id)); + rb_devices[device_id] = std::move(rbm); + device_ids.insert(device_id); + } + + void reset() { + rb_devices.clear(); + rb_devices.resize(DEVICE_ID_MAX); + device_ids.clear(); + } + + auto get_block_size() const { + assert(device_ids.size()); + return rb_devices[*device_ids.begin()]->get_block_size(); + } + + const seastore_meta_t &get_meta() const { + assert(device_ids.size()); + return rb_devices[*device_ids.begin()]->get_meta(); + } + +private: + bool has_device(device_id_t id) const { + assert(id <= DEVICE_ID_MAX_VALID); + return device_ids.count(id) >= 1; + } + + std::vector rb_devices; + std::set device_ids; +}; + +using RBMDeviceGroupRef = std::unique_ptr; + +} diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index d924050351c..3fffe202a54 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -629,17 +629,26 @@ TransactionManagerRef make_transaction_manager( auto cache = std::make_unique(*epm); auto lba_manager = lba_manager::create_lba_manager(*cache); auto sms = std::make_unique(); + auto rbs = std::make_unique(); auto backref_manager = create_backref_manager(*cache); auto p_backend_type = primary_device->get_backend_type(); if (p_backend_type == backend_type_t::SEGMENTED) { sms->add_segment_manager(static_cast(primary_device)); + } else { + auto rbm = std::make_unique( + static_cast(primary_device), ""); + rbs->add_rb_manager(std::move(rbm)); } for (auto &p_dev : secondary_devices) { - ceph_assert(p_dev->get_backend_type() == backend_type_t::SEGMENTED); - sms->add_segment_manager(static_cast(p_dev)); + if (p_dev->get_backend_type() == backend_type_t::SEGMENTED) { + sms->add_segment_manager(static_cast(p_dev)); + } else { + auto rbm = std::make_unique(static_cast(p_dev), ""); + rbs->add_rb_manager(std::move(rbm)); + } } auto journal_type = p_backend_type; @@ -681,6 +690,7 @@ TransactionManagerRef make_transaction_manager( journal_type, roll_start, roll_size); AsyncCleanerRef cleaner; + JournalRef journal; if (journal_type == journal_type_t::SEGMENTED) { cleaner = SegmentCleaner::create( @@ -691,19 +701,14 @@ TransactionManagerRef make_transaction_manager( auto segment_cleaner = static_cast(cleaner.get()); cache->set_segment_provider(*segment_cleaner); segment_cleaner->set_journal_trimmer(*journal_trimmer); - } else { - cleaner = RBMCleaner::create( - *backref_manager, - cleaner_is_detailed); - } - - JournalRef journal; - if (journal_type == journal_type_t::SEGMENTED) { - auto segment_cleaner = static_cast(cleaner.get()); journal = journal::make_segmented( *segment_cleaner, *journal_trimmer); } else { + cleaner = RBMCleaner::create( + std::move(rbs), + *backref_manager, + cleaner_is_detailed); journal = journal::make_circularbounded( *journal_trimmer, static_cast(primary_device), diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index c0306db4409..c283d49ea3e 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -399,6 +399,11 @@ struct transaction_manager_test_t : } bool check_usage() { + std::string j_type = GetParam(); + if (j_type == "circularbounded") { + // TODO: add check_usage for RBM + return true; + } return epm->check_usage(); } diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 1ea50e5160e..4b79538e34e 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -27,16 +27,23 @@ class EphemeralTestState { protected: segment_manager::EphemeralSegmentManagerRef segment_manager; std::list secondary_segment_managers; - std::unique_ptr rb_device; + random_block_device::RBMDeviceRef rb_device; + std::list secondary_rb_devices; journal_type_t journal_type; - EphemeralTestState(std::size_t num_segment_managers) { - assert(num_segment_managers > 0); - secondary_segment_managers.resize(num_segment_managers - 1); + EphemeralTestState(std::size_t num_device_managers) { + assert(num_device_managers > 0); + secondary_segment_managers.resize(num_device_managers - 1); + secondary_rb_devices.resize(num_device_managers - 1); } std::size_t get_num_devices() const { - return secondary_segment_managers.size() + 1; + if (journal_type == journal_type_t::SEGMENTED) { + return secondary_segment_managers.size() + 1; + } else { + assert(journal_type == journal_type_t::RANDOM_BLOCK); + return secondary_rb_devices.size() + 1; + } } virtual void _init() = 0; @@ -57,9 +64,11 @@ protected: LOG_PREFIX(EphemeralTestState::restart_fut); SUBINFO(test, "begin ..."); return teardown().then([this] { - segment_manager->remount(); - for (auto &sec_sm : secondary_segment_managers) { - sec_sm->remount(); + if (journal_type == journal_type_t::SEGMENTED) { + segment_manager->remount(); + for (auto &sec_sm : secondary_segment_managers) { + sec_sm->remount(); + } } _init(); return _mount().handle_error(crimson::ct_error::assert_all{}); @@ -117,8 +126,9 @@ protected: seastar::future<> randomblock_setup() { + LOG_PREFIX(EphemeralTestState::randomblock_setup); rb_device = random_block_device::create_test_ephemeral( - journal::DEFAULT_TEST_CBJOURNAL_SIZE, 0); + journal::DEFAULT_TEST_CBJOURNAL_SIZE, journal::DEFAULT_TEST_CBJOURNAL_SIZE); return rb_device->mount().handle_error(crimson::ct_error::assert_all{} ).then([this]() { device_config_t config = get_rbm_ephemeral_device_config(0, 1); @@ -133,8 +143,8 @@ protected: seastar::future<> tm_setup( journal_type_t type = journal_type_t::SEGMENTED) { LOG_PREFIX(EphemeralTestState::tm_setup); - SUBINFO(test, "begin with {} devices ...", get_num_devices()); journal_type = type; + SUBINFO(test, "begin with {} devices ...", get_num_devices()); // FIXME: should not initialize segment_manager with circularbounded-journal if (journal_type == journal_type_t::SEGMENTED) { return segment_setup(); @@ -153,6 +163,9 @@ protected: sec_sm.reset(); } rb_device.reset(); + for (auto &sec_rb : secondary_rb_devices) { + sec_rb.reset(); + } SUBINFO(test, "finish"); }); } @@ -172,15 +185,17 @@ protected: virtual void _init() override { std::vector sec_devices; - for (auto &sec_sm : secondary_segment_managers) { - sec_devices.emplace_back(sec_sm.get()); - } if (journal_type == journal_type_t::RANDOM_BLOCK) { // FIXME: should not initialize segment_manager with circularbounded-journal // FIXME: no secondary device in the single device test - sec_devices.emplace_back(segment_manager.get()); + for (auto &sec_rb : secondary_rb_devices) { + sec_devices.emplace_back(sec_rb.get()); + } tm = make_transaction_manager(rb_device.get(), sec_devices, true); } else { + for (auto &sec_sm : secondary_segment_managers) { + sec_devices.emplace_back(sec_sm.get()); + } tm = make_transaction_manager(segment_manager.get(), sec_devices, true); } epm = tm->get_epm(); -- 2.39.5