From 138f06cae5a34e256bb378adb090fdd0c67db57e Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 22 Apr 2021 16:56:08 -0700 Subject: [PATCH] crimson/os/seastore: refactor SegmentManager reference ownership Gives SeaStore ownership over SegmentManager and rearranges mkfs/mount. Replaces mkfs_config_t/mount_config_t with config params. Signed-off-by: Samuel Just --- src/common/options/crimson.yaml.in | 17 +++++ src/crimson/os/seastore/seastore.cc | 17 ++++- src/crimson/os/seastore/seastore.h | 5 +- src/crimson/os/seastore/segment_manager.h | 13 ++++ .../os/seastore/segment_manager/block.cc | 36 ++++++---- .../os/seastore/segment_manager/block.h | 28 ++------ .../os/seastore/segment_manager/ephemeral.h | 8 +++ .../os/seastore/transaction_manager.cc | 2 +- src/crimson/tools/store-nbd.cc | 65 +++++++++---------- .../onode_tree/test_fltree_onode_manager.cc | 27 ++++---- .../seastore/transaction_manager_test_state.h | 47 ++++++++++++-- 11 files changed, 174 insertions(+), 91 deletions(-) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index eb4c9d74214..38fd8c03247 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -23,3 +23,20 @@ options: type: str level: advanced desc: CPU cores on which alienstore threads will run +- name: seastore_segment_size + type: size + desc: Segment size to use for SegmentManager + level: advanced + default: 64_M +- name: seastore_device_size + type: size + desc: Total size to use for SegmentManager block file if created + level: dev + default: 100_G +- name: seastore_block_create + type: bool + level: dev + desc: Create SegmentManager file if it doesn't exist + default: true + see_also: + - seastore_device_size diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 4fb577113a7..62101430abf 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -46,8 +46,10 @@ seastar::future<> SeaStore::stop() seastar::future<> SeaStore::mount() { - return transaction_manager->mount( - ).handle_error( + return segment_manager->mount( + ).safe_then([this] { + return transaction_manager->mount(); + }).handle_error( crimson::ct_error::assert_all{ "Invalid error in SeaStore::mount" } @@ -61,8 +63,15 @@ seastar::future<> SeaStore::umount() seastar::future<> SeaStore::mkfs(uuid_d new_osd_fsid) { - return transaction_manager->mkfs( + return segment_manager->mkfs( + seastore_meta_t{new_osd_fsid} ).safe_then([this] { + return segment_manager->mount(); + }).safe_then([this] { + return transaction_manager->mkfs(); + }).safe_then([this] { + return transaction_manager->mount(); + }).safe_then([this] { return seastar::do_with( transaction_manager->create_transaction(), [this](auto &t) { @@ -77,6 +86,8 @@ seastar::future<> SeaStore::mkfs(uuid_d new_osd_fsid) std::move(t)); }); }); + }).safe_then([this] { + return stop(); }).handle_error( crimson::ct_error::assert_all{ "Invalid error in SeaStore::mkfs" diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index 1eb568ebd0e..e673e4fff74 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -33,10 +33,12 @@ class SeaStore final : public FuturizedStore { public: SeaStore( + SegmentManagerRef sm, TransactionManagerRef tm, CollectionManagerRef cm, OnodeManagerRef om - ) : transaction_manager(std::move(tm)), + ) : segment_manager(std::move(sm)), + transaction_manager(std::move(tm)), collection_manager(std::move(cm)), onode_manager(std::move(om)) {} @@ -239,6 +241,7 @@ private: const std::optional& start, OMapManager::omap_list_config_t config); + SegmentManagerRef segment_manager; TransactionManagerRef transaction_manager; CollectionManagerRef collection_manager; OnodeManagerRef onode_manager; diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h index 61c6509d19f..192564a4b18 100644 --- a/src/crimson/os/seastore/segment_manager.h +++ b/src/crimson/os/seastore/segment_manager.h @@ -79,6 +79,19 @@ constexpr size_t PADDR_SIZE = sizeof(paddr_t); class SegmentManager { public: + using access_ertr = crimson::errorator< + crimson::ct_error::input_output_error, + crimson::ct_error::permission_denied, + crimson::ct_error::enoent>; + + using mount_ertr = access_ertr; + using mount_ret = access_ertr::future<>; + virtual mount_ret mount() = 0; + + using mkfs_ertr = access_ertr; + using mkfs_ret = mkfs_ertr::future<>; + virtual mkfs_ret mkfs(seastore_meta_t meta) = 0; + using open_ertr = crimson::errorator< crimson::ct_error::input_output_error, crimson::ct_error::invarg, diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index 36e890894ce..0094e58e2c2 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -4,6 +4,7 @@ #include #include +#include "crimson/common/config_proxy.h" #include "crimson/common/log.h" #include "include/buffer.h" @@ -92,31 +93,40 @@ SegmentStateTracker::read_in( static block_sm_superblock_t make_superblock( - const BlockSegmentManager::mkfs_config_t &config, + seastore_meta_t meta, const seastar::stat_data &data) { + using crimson::common::get_conf; + + auto config_size = get_conf( + "seastore_device_size"); + logger().debug( "{}: size {}, block_size {}, allocated_size {}, configured_size {}", __func__, data.size, data.block_size, data.allocated_size, - config.total_size); - size_t size = (data.size == 0) ? config.total_size : data.size; - size_t raw_segments = size / config.segment_size; + config_size); + + size_t size = (data.size == 0) ? config_size : data.size; + + auto config_segment_size = get_conf( + "seastore_segment_size"); + size_t raw_segments = size / config_segment_size; size_t tracker_size = SegmentStateTracker::get_raw_size( raw_segments, data.block_size); size_t segments = (size - tracker_size - data.block_size) - / config.segment_size; + / config_segment_size; return block_sm_superblock_t{ size, - config.segment_size, + config_segment_size, data.block_size, segments, data.block_size, tracker_size + data.block_size, - config.meta + meta }; } @@ -268,10 +278,10 @@ BlockSegmentManager::~BlockSegmentManager() { } -BlockSegmentManager::mount_ret BlockSegmentManager::mount(const mount_config_t& config) +BlockSegmentManager::mount_ret BlockSegmentManager::mount() { return open_device( - config.path, seastar::open_flags::rw | seastar::open_flags::dsync + device_path, seastar::open_flags::rw | seastar::open_flags::dsync ).safe_then([=](auto p) { device = std::move(p.first); auto sd = p.second; @@ -295,7 +305,7 @@ BlockSegmentManager::mount_ret BlockSegmentManager::mount(const mount_config_t& }); } -BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs(mkfs_config_t config) +BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs(seastore_meta_t meta) { return seastar::do_with( seastar::file{}, @@ -304,11 +314,11 @@ BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs(mkfs_config_t config) std::unique_ptr(), [=](auto &device, auto &stat, auto &sb, auto &tracker) { return open_device( - config.path, seastar::open_flags::rw - ).safe_then([&, config](auto p) { + device_path, seastar::open_flags::rw + ).safe_then([&, meta](auto p) { device = p.first; stat = p.second; - sb = make_superblock(config, stat); + sb = make_superblock(meta, stat); return write_superblock(device, sb); }).safe_then([&] { logger().debug("BlockSegmentManager::mkfs: superblock written"); diff --git a/src/crimson/os/seastore/segment_manager/block.h b/src/crimson/os/seastore/segment_manager/block.h index bfa4bd7e192..6cadbf018ff 100644 --- a/src/crimson/os/seastore/segment_manager/block.h +++ b/src/crimson/os/seastore/segment_manager/block.h @@ -132,35 +132,16 @@ public: */ class BlockSegmentManager final : public SegmentManager { public: - using access_ertr = crimson::errorator< - crimson::ct_error::input_output_error, - crimson::ct_error::permission_denied, - crimson::ct_error::enoent>; - - - struct mount_config_t { - std::string path; - }; - using mount_ertr = access_ertr; - using mount_ret = access_ertr::future<>; - mount_ret mount(const mount_config_t&); - - struct mkfs_config_t { - std::string path; - size_t segment_size = 0; - size_t total_size = 0; - seastore_meta_t meta; - }; - using mkfs_ertr = access_ertr; - using mkfs_ret = mkfs_ertr::future<>; - static mkfs_ret mkfs(mkfs_config_t); + mount_ret mount() final; + + mkfs_ret mkfs(seastore_meta_t) final; using close_ertr = crimson::errorator< crimson::ct_error::input_output_error >; close_ertr::future<> close(); - BlockSegmentManager() = default; + BlockSegmentManager(const std::string &path) : device_path(path) {} ~BlockSegmentManager(); open_ertr::future open(segment_id_t id) final; @@ -193,6 +174,7 @@ private: using segment_state_t = Segment::segment_state_t; + std::string device_path; std::unique_ptr tracker; block_sm_superblock_t superblock; seastar::file device; diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.h b/src/crimson/os/seastore/segment_manager/ephemeral.h index 9f19cb4d0c7..2096fab3910 100644 --- a/src/crimson/os/seastore/segment_manager/ephemeral.h +++ b/src/crimson/os/seastore/segment_manager/ephemeral.h @@ -75,6 +75,14 @@ public: crimson::ct_error::erange>; init_ertr::future<> init(); + mount_ret mount() { + return mount_ertr::now(); + } + + mkfs_ret mkfs(seastore_meta_t) { + return mkfs_ertr::now(); + } + open_ertr::future open(segment_id_t id) final; release_ertr::future<> release(segment_id_t id) final; diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 1862de85658..62ea036ea3b 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -55,7 +55,7 @@ TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs() }); }); }).safe_then([this] { - return journal->close(); + return close(); }); } diff --git a/src/crimson/tools/store-nbd.cc b/src/crimson/tools/store-nbd.cc index fb485e57430..065c94c7be2 100644 --- a/src/crimson/tools/store-nbd.cc +++ b/src/crimson/tools/store-nbd.cc @@ -38,6 +38,9 @@ #include #include +#include "crimson/common/config_proxy.h" +#include "crimson/common/perf_counters_collection.h" + #include "crimson/os/seastore/cache.h" #include "crimson/os/seastore/segment_cleaner.h" #include "crimson/os/seastore/segment_manager.h" @@ -73,8 +76,6 @@ public: std::string type; bool mkfs = false; std::optional path; - size_t segment_size; - size_t total_device_size; void populate_options( po::options_description &desc) @@ -86,18 +87,6 @@ public: ->notifier([this](auto s) { type = s; }), "Backend to use, options are transaction_manager" ) - ("segment-size", - po::value() - ->default_value(16ul << 20 /* 16MB */) - ->notifier([this](auto s) { segment_size = s; }), - "Total working set size" - ) - ("total-device-size", - po::value() - ->default_value(10ul << 30 /* 10G */) - ->notifier([this](auto s) { total_device_size = s; }), - "Size of writes" - ) ("device-path", po::value() ->required() @@ -374,20 +363,29 @@ int main(int argc, char** argv) } sc.run([=] { - auto backend = get_backend(backend_config); - return seastar::do_with( - NBDHandler(*backend, nbd_config), - std::move(backend), - [](auto &nbd, auto &backend) { - return backend->mount( - ).then([&] { - logger().debug("Running nbd server..."); - return nbd.run(); - }).then([&] { - return backend->close(); + return crimson::common::sharded_conf( + ).start(EntityName{}, string_view{"ceph"} + ).then([=] { + auto backend = get_backend(backend_config); + return seastar::do_with( + NBDHandler(*backend, nbd_config), + std::move(backend), + [](auto &nbd, auto &backend) { + return backend->mount( + ).then([&] { + logger().debug("Running nbd server..."); + return nbd.run(); + }).then([&] { + return backend->close(); + }); }); + }).then([=] { + return crimson::common::sharded_perf_coll().stop().then([] { + return crimson::common::sharded_conf().stop(); }); + }); }); + sc.stop(); } @@ -663,16 +661,15 @@ public: assert(config.path); segment_manager = std::make_unique< segment_manager::block::BlockSegmentManager - >(); + >(*config.path); logger().debug("mkfs"); - BlockSegmentManager::mkfs_config_t block_config{ - *config.path, config.segment_size, config.total_device_size - }; - block_config.meta.seastore_id.generate_random(); - return segment_manager->mkfs(std::move(block_config) + seastore_meta_t meta; + meta.seastore_id.generate_random(); + return segment_manager->mkfs( + std::move(meta) ).safe_then([this] { logger().debug(""); - return segment_manager->mount({ *config.path }); + return segment_manager->mount(); }).safe_then([this] { init(); logger().debug("tm mkfs"); @@ -699,8 +696,8 @@ public: ).then([this] { segment_manager = std::make_unique< segment_manager::block::BlockSegmentManager - >(); - return segment_manager->mount({ *config.path }); + >(*config.path); + return segment_manager->mount(); }).safe_then([this] { init(); return tm->mount(); diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc index 96c8384c317..455a4c4df12 100644 --- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc +++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc @@ -82,18 +82,21 @@ struct fltree_onode_manager_test_t virtual seastar::future<> _mkfs() final { return TMTestState::_mkfs( ).then([this] { - return seastar::do_with( - tm->create_transaction(), - [this](auto &t) { - return manager->mkfs(*t - ).safe_then([this, &t] { - return tm->submit_transaction(std::move(t)); - }).handle_error( - crimson::ct_error::assert_all{ - "Invalid error in _mkfs" - } - ); - }); + return tm->mount( + ).safe_then([this] { + return seastar::do_with( + tm->create_transaction(), + [this](auto &t) { + return manager->mkfs(*t + ).safe_then([this, &t] { + return tm->submit_transaction(std::move(t)); + }); + }); + }).safe_then([this] { + return tm->close(); + }).handle_error( + crimson::ct_error::assert_all{"Invalid error in _mkfs"} + ); }); } diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index b2e37f7c3f9..2fa8ebe6593 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -91,12 +91,12 @@ auto get_transaction_manager( } auto get_seastore( - SegmentManager &segment_manager + SegmentManagerRef sm ) { - - auto tm = get_transaction_manager(segment_manager); + auto tm = get_transaction_manager(*sm); auto cm = std::make_unique(*tm); return std::make_unique( + std::move(sm), std::move(tm), std::move(cm), std::make_unique(*tm)); @@ -149,6 +149,44 @@ protected: } }; +class TestSegmentManagerWrapper : public SegmentManager { + SegmentManager &sm; +public: + TestSegmentManagerWrapper(SegmentManager &sm) : sm(sm) {} + + mount_ret mount() final { + return mount_ertr::now(); // we handle this above + } + + mkfs_ret mkfs(seastore_meta_t c) final { + return mkfs_ertr::now(); // we handle this above + } + + + open_ertr::future open(segment_id_t id) final { + return sm.open(id); + } + + release_ertr::future<> release(segment_id_t id) final { + return sm.release(id); + } + + read_ertr::future<> read( + paddr_t addr, size_t len, ceph::bufferptr &out) final { + return sm.read(addr, len, out); + } + + size_t get_size() const final { return sm.get_size(); } + segment_off_t get_block_size() const final { return sm.get_block_size(); } + segment_off_t get_segment_size() const final { + return sm.get_segment_size(); + } + const seastore_meta_t &get_meta() const final { + return sm.get_meta(); + } + ~TestSegmentManagerWrapper() final {} +}; + class SeaStoreTestState : public EphemeralTestState { protected: std::unique_ptr seastore; @@ -156,7 +194,8 @@ protected: SeaStoreTestState() : EphemeralTestState() {} virtual void _init() { - seastore = get_seastore(*segment_manager); + seastore = get_seastore( + std::make_unique(*segment_manager)); } virtual void _destroy() { -- 2.39.5