From eba5c18164fb60baf33d947185e733c289ea9d54 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 23 Mar 2022 11:33:51 +0800 Subject: [PATCH] crimson/os/seastore: replace SegmentManager by Device at SeaStore level Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/CMakeLists.txt | 1 + src/crimson/os/seastore/device.cc | 44 +++++ src/crimson/os/seastore/device.h | 70 ++++++++ src/crimson/os/seastore/seastore.cc | 157 +++++++++--------- src/crimson/os/seastore/seastore.h | 9 +- src/crimson/os/seastore/seastore_types.cc | 3 +- src/crimson/os/seastore/segment_manager.cc | 24 --- src/crimson/os/seastore/segment_manager.h | 62 +------ .../os/seastore/segment_manager/block.cc | 4 +- .../os/seastore/segment_manager/block.h | 8 +- .../os/seastore/segment_manager/ephemeral.h | 8 +- .../os/seastore/segment_manager/zns.cc | 11 +- src/crimson/os/seastore/segment_manager/zns.h | 4 +- .../os/seastore/transaction_manager.cc | 7 +- src/crimson/os/seastore/transaction_manager.h | 21 ++- src/crimson/tools/store_nbd/tm_driver.cc | 63 ++++--- src/crimson/tools/store_nbd/tm_driver.h | 6 +- .../onode_tree/test_fltree_onode_manager.cc | 2 +- .../seastore/transaction_manager_test_state.h | 8 +- 19 files changed, 260 insertions(+), 252 deletions(-) create mode 100644 src/crimson/os/seastore/device.cc diff --git a/src/crimson/os/seastore/CMakeLists.txt b/src/crimson/os/seastore/CMakeLists.txt index 96e9384b6c173..a10d1c677d8db 100644 --- a/src/crimson/os/seastore/CMakeLists.txt +++ b/src/crimson/os/seastore/CMakeLists.txt @@ -39,6 +39,7 @@ set(crimson_seastore_srcs journal/segmented_journal.cc journal/segment_allocator.cc journal.cc + device.cc ../../../test/crimson/seastore/test_block.cc ${PROJECT_SOURCE_DIR}/src/os/Transaction.cc ) diff --git a/src/crimson/os/seastore/device.cc b/src/crimson/os/seastore/device.cc new file mode 100644 index 0000000000000..3997216fc0ccf --- /dev/null +++ b/src/crimson/os/seastore/device.cc @@ -0,0 +1,44 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- +// vim: ts=8 sw=2 smarttab + +#include "device.h" + +#include "segment_manager.h" + +namespace crimson::os::seastore { + +std::ostream& operator<<(std::ostream& out, const device_spec_t& ds) +{ + return out << "device_spec(" + << "magic=" << ds.magic + << ", device_type=" << ds.dtype + << ", device_id=" << device_id_printer_t{ds.id} + << ")"; +} + +std::ostream& operator<<(std::ostream& out, const device_config_t& conf) +{ + out << "device_config_t(" + << "major_dev=" << conf.major_dev + << ", magic=" << conf.magic + << ", device_type=" << conf.dtype + << ", device_id=" << device_id_printer_t{conf.device_id} + << ", meta=" << conf.meta + << ", secondary("; + for (const auto& [k, v] : conf.secondary_devices) { + out << std::to_string(k) << ": " << v; + } + return out << "))"; +} + +seastar::future +Device::make_device(const std::string& device) +{ + // TODO: configure device type + return SegmentManager::get_segment_manager(device + ).then([](DeviceRef ret) { + return ret; + }); +} + +} diff --git a/src/crimson/os/seastore/device.h b/src/crimson/os/seastore/device.h index 33a97a6d70c78..3b365eb7dcef3 100644 --- a/src/crimson/os/seastore/device.h +++ b/src/crimson/os/seastore/device.h @@ -8,9 +8,44 @@ #include "include/buffer_fwd.h" #include "crimson/common/errorator.h" +#include "crimson/os/seastore/seastore_types.h" namespace crimson::os::seastore { +using magic_t = uint64_t; + +struct device_spec_t{ + magic_t magic; + device_type_t dtype; + device_id_t id; + DENC(device_spec_t, v, p) { + DENC_START(1, 1, p); + denc(v.magic, p); + denc(v.dtype, p); + denc(v.id, p); + DENC_FINISH(p); + } +}; + +std::ostream& operator<<(std::ostream&, const device_spec_t&); + +using secondary_device_set_t = + std::map; + +struct device_config_t { + bool major_dev = false; + magic_t magic = 0; + device_type_t dtype = device_type_t::NONE; + device_id_t device_id = DEVICE_ID_NULL; + seastore_meta_t meta; + secondary_device_set_t secondary_devices; +}; + +std::ostream& operator<<(std::ostream&, const device_config_t&); + +class Device; +using DeviceRef = std::unique_ptr; + /** * Device * @@ -18,10 +53,39 @@ namespace crimson::os::seastore { */ class Device { public: + virtual ~Device() {} + virtual device_id_t get_device_id() const = 0; + virtual magic_t get_magic() const = 0; + + virtual device_type_t get_device_type() const = 0; + + virtual const seastore_meta_t &get_meta() const = 0; + virtual seastore_off_t get_block_size() const = 0; + virtual std::size_t get_size() const = 0; + + virtual secondary_device_set_t& get_secondary_devices() = 0; + + using access_ertr = crimson::errorator< + crimson::ct_error::input_output_error, + crimson::ct_error::permission_denied, + crimson::ct_error::enoent>; + + using mkfs_ertr = access_ertr; + using mkfs_ret = mkfs_ertr::future<>; + virtual mkfs_ret mkfs(device_config_t) = 0; + + using mount_ertr = access_ertr; + using mount_ret = access_ertr::future<>; + virtual mount_ret mount() = 0; + + using close_ertr = crimson::errorator< + crimson::ct_error::input_output_error>; + virtual close_ertr::future<> close() = 0; + using read_ertr = crimson::errorator< crimson::ct_error::input_output_error, crimson::ct_error::invarg, @@ -43,6 +107,12 @@ public: return read_ertr::make_ready_future(std::move(*ptrref)); }); } + + static seastar::future make_device(const std::string &device); }; } + +WRITE_CLASS_DENC( + crimson::os::seastore::device_spec_t +) diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 9be8bcd5f151f..4771c95f92e9a 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -23,12 +23,9 @@ #include "crimson/os/futurized_collection.h" #include "crimson/os/seastore/segment_cleaner.h" -#include "crimson/os/seastore/segment_manager.h" -#include "crimson/os/seastore/segment_manager/block.h" #include "crimson/os/seastore/collection_manager/flat_collection_manager.h" #include "crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h" #include "crimson/os/seastore/omap_manager/btree/btree_omap_manager.h" -#include "crimson/os/seastore/segment_manager/ephemeral.h" #include "crimson/os/seastore/onode_manager.h" #include "crimson/os/seastore/object_data_handler.h" @@ -118,13 +115,13 @@ using crimson::common::get_conf; SeaStore::SeaStore( const std::string& root, MDStoreRef mdstore, - SegmentManagerRef sm, + DeviceRef dev, TransactionManagerRef tm, CollectionManagerRef cm, OnodeManagerRef om) : root(root), mdstore(std::move(mdstore)), - segment_manager(std::move(sm)), + device(std::move(dev)), transaction_manager(std::move(tm)), collection_manager(std::move(cm)), onode_manager(std::move(om)), @@ -136,14 +133,14 @@ SeaStore::SeaStore( SeaStore::SeaStore( const std::string& root, - SegmentManagerRef sm, + DeviceRef dev, TransactionManagerRef tm, CollectionManagerRef cm, OnodeManagerRef om) : SeaStore( root, std::make_unique(root), - std::move(sm), std::move(tm), std::move(cm), std::move(om)) {} + std::move(dev), std::move(tm), std::move(cm), std::move(om)) {} SeaStore::~SeaStore() = default; @@ -187,25 +184,26 @@ seastar::future<> SeaStore::stop() SeaStore::mount_ertr::future<> SeaStore::mount() { - return segment_manager->mount( + secondaries.clear(); + return device->mount( ).safe_then([this] { - transaction_manager->add_segment_manager(segment_manager.get(), true); - auto sec_devices = segment_manager->get_secondary_devices(); + transaction_manager->add_device(device.get(), true); + auto sec_devices = device->get_secondary_devices(); return crimson::do_for_each(sec_devices, [this](auto& device_entry) { device_id_t id = device_entry.first; magic_t magic = device_entry.second.magic; device_type_t dtype = device_entry.second.dtype; std::ostringstream oss; oss << root << "/block." << dtype << "." << std::to_string(id); - auto sm = std::make_unique< - segment_manager::block::BlockSegmentManager>(oss.str()); - return sm->mount().safe_then( - [this, sm=std::move(sm), magic]() mutable { - boost::ignore_unused(magic); // avoid clang warning; - assert(sm->get_magic() == magic); - transaction_manager->add_segment_manager(sm.get(), false); - secondaries.emplace_back(std::move(sm)); - return seastar::now(); + return Device::make_device(oss.str() + ).then([this, magic](DeviceRef sec_dev) { + return sec_dev->mount( + ).safe_then([this, sec_dev=std::move(sec_dev), magic]() mutable { + boost::ignore_unused(magic); // avoid clang warning; + assert(sec_dev->get_magic() == magic); + transaction_manager->add_device(sec_dev.get(), false); + secondaries.emplace_back(std::move(sec_dev)); + }); }); }); }).safe_then([this] { @@ -223,11 +221,12 @@ seastar::future<> SeaStore::umount() ).safe_then([this] { return crimson::do_for_each( secondaries, - [](auto& sm) -> SegmentManager::close_ertr::future<> { - return sm->close(); + [](auto& sec_dev) -> SegmentManager::close_ertr::future<> + { + return sec_dev->close(); }); }).safe_then([this] { - return segment_manager->close(); + return device->close(); }).handle_error( crimson::ct_error::assert_all{ "Invalid error in SeaStore::umount" @@ -267,13 +266,13 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) LOG_PREFIX(SeaStore::mkfs); DEBUG("root: {}", root); if (!root.empty()) { - fut = seastar::open_directory(root).then( - [this, &sds, new_osd_fsid](seastar::file rdir) mutable { + fut = seastar::open_directory(root + ).then([this, &sds, new_osd_fsid](seastar::file rdir) mutable { std::unique_ptr root_f = std::make_unique(std::move(rdir)); auto sub = root_f->list_directory( - [this, &sds, new_osd_fsid](auto de) mutable - -> seastar::future<> { + [this, &sds, new_osd_fsid](auto de) mutable -> seastar::future<> + { LOG_PREFIX(SeaStore::mkfs); DEBUG("found file: {}", de.name); if (de.name.find("block.") == 0 @@ -288,42 +287,36 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) return seastar::now(); } auto id = std::stoi(entry_name.substr(dtype_end + 1)); - auto sm = std::make_unique< - segment_manager::block::BlockSegmentManager - >(root + "/" + entry_name); - magic_t magic = (magic_t)std::rand(); - sds.emplace( - (device_id_t)id, - device_spec_t{ - magic, - dtype, - (device_id_t)id}); - return sm->mkfs( - segment_manager_config_t{ - false, - magic, - dtype, + std::ostringstream oss; + oss << root << "/" << entry_name; + return Device::make_device(oss.str() + ).then([this, &sds, id, dtype, new_osd_fsid](DeviceRef sec_dev) { + magic_t magic = (magic_t)std::rand(); + sds.emplace( (device_id_t)id, - seastore_meta_t{new_osd_fsid}, - secondary_device_set_t()} - ).safe_then([this, sm=std::move(sm), id]() mutable { - LOG_PREFIX(SeaStore::mkfs); - DEBUG("mkfs: finished for segment manager {}", id); - secondaries.emplace_back(std::move(sm)); - return seastar::now(); - }).handle_error(crimson::ct_error::assert_all{"not possible"}); + device_spec_t{magic, dtype, (device_id_t)id}); + return sec_dev->mkfs(device_config_t{ + false, + magic, + dtype, + (device_id_t)id, + seastore_meta_t{new_osd_fsid}, + secondary_device_set_t()} + ).safe_then([this, sec_dev=std::move(sec_dev), id]() mutable { + LOG_PREFIX(SeaStore::mkfs); + DEBUG("mkfs: finished for device {}", id); + secondaries.emplace_back(std::move(sec_dev)); + }).handle_error(crimson::ct_error::assert_all{"not possible"}); + }); } - return seastar::now(); - }); - return sub.done().then( - [root_f=std::move(root_f)] { return seastar::now(); }); + return sub.done().then([root_f=std::move(root_f)] {}); }); } return fut.then([this, &sds, new_osd_fsid] { - return segment_manager->mkfs( - segment_manager_config_t{ + return device->mkfs( + device_config_t{ true, (magic_t)std::rand(), device_type_t::SEGMENTED, @@ -332,23 +325,23 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) sds} ); }).safe_then([this] { - return crimson::do_for_each(secondaries, [this](auto& sec_sm) { - return sec_sm->mount().safe_then([this, &sec_sm] { - transaction_manager->add_segment_manager(sec_sm.get(), false); + return crimson::do_for_each(secondaries, [this](auto& sec_dev) { + return sec_dev->mount().safe_then([this, &sec_dev] { + transaction_manager->add_device(sec_dev.get(), false); return seastar::now(); }); }); }); }).safe_then([this] { - return segment_manager->mount(); + return device->mount(); }).safe_then([this] { - transaction_manager->add_segment_manager(segment_manager.get(), true); + transaction_manager->add_device(device.get(), true); return transaction_manager->mkfs(); }).safe_then([this] { - for (auto& sec_sm : secondaries) { - transaction_manager->add_segment_manager(sec_sm.get(), false); + for (auto& sec_dev : secondaries) { + transaction_manager->add_device(sec_dev.get(), false); } - transaction_manager->add_segment_manager(segment_manager.get(), true); + transaction_manager->add_device(device.get(), true); return transaction_manager->mount(); }).safe_then([this] { return repeat_eagain([this] { @@ -581,7 +574,7 @@ SeaStore::get_attr_errorator::future SeaStore::get_attr( return _omap_get_value( t, layout.xattr_root.get( - onode.get_metadata_hint(segment_manager->get_block_size())), + onode.get_metadata_hint(device->get_block_size())), name); } ).handle_error(crimson::ct_error::input_output_error::handle([FNAME] { @@ -643,7 +636,7 @@ seastar::future SeaStore::stat( struct stat st; auto &olayout = onode.get_layout(); st.st_size = olayout.size; - st.st_blksize = segment_manager->get_block_size(); + st.st_blksize = device->get_block_size(); st.st_blocks = (st.st_size + st.st_blksize - 1) / st.st_blksize; st.st_nlink = 1; DEBUGT("cid {}, oid {}, return size {}", t, c->get_cid(), oid, st.st_size); @@ -680,7 +673,7 @@ SeaStore::omap_get_values( op_type_t::OMAP_GET_VALUES, [this, keys](auto &t, auto &onode) { omap_root_t omap_root = onode.get_layout().omap_root.get( - onode.get_metadata_hint(segment_manager->get_block_size())); + onode.get_metadata_hint(device->get_block_size())); return _omap_get_values( t, std::move(omap_root), @@ -759,7 +752,7 @@ SeaStore::_omap_list_ret SeaStore::_omap_list( OMapManager::omap_list_config_t config) const { auto root = omap_root.get( - onode.get_metadata_hint(segment_manager->get_block_size())); + onode.get_metadata_hint(device->get_block_size())); if (root.is_null()) { return seastar::make_ready_future<_omap_list_bare_ret>( true, omap_values_t{} @@ -1235,13 +1228,13 @@ SeaStore::_omap_set_kvs( { return seastar::do_with( BtreeOMapManager(*transaction_manager), - omap_root.get(onode->get_metadata_hint(segment_manager->get_block_size())), + omap_root.get(onode->get_metadata_hint(device->get_block_size())), [&, keys=std::move(kvs)](auto &omap_manager, auto &root) { tm_iertr::future<> maybe_create_root = !root.is_null() ? tm_iertr::now() : omap_manager.initialize_omap( - t, onode->get_metadata_hint(segment_manager->get_block_size()) + t, onode->get_metadata_hint(device->get_block_size()) ).si_then([&root](auto new_root) { root = new_root; }); @@ -1292,14 +1285,14 @@ SeaStore::tm_ret SeaStore::_omap_clear( LOG_PREFIX(SeaStore::_omap_clear); DEBUGT("{} {} keys", *ctx.transaction, *onode); if (auto omap_root = onode->get_layout().omap_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())); + onode->get_metadata_hint(device->get_block_size())); omap_root.is_null()) { return seastar::now(); } else { return seastar::do_with( BtreeOMapManager(*transaction_manager), onode->get_layout().omap_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())), + onode->get_metadata_hint(device->get_block_size())), [&ctx, &onode]( auto &omap_manager, auto &omap_root) { @@ -1324,14 +1317,14 @@ SeaStore::tm_ret SeaStore::_omap_rmkeys( LOG_PREFIX(SeaStore::_omap_rmkeys); DEBUGT("{} {} keys", *ctx.transaction, *onode, keys.size()); auto omap_root = onode->get_layout().omap_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())); + onode->get_metadata_hint(device->get_block_size())); if (omap_root.is_null()) { return seastar::now(); } else { return seastar::do_with( BtreeOMapManager(*transaction_manager), onode->get_layout().omap_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())), + onode->get_metadata_hint(device->get_block_size())), std::move(keys), [&ctx, &onode]( auto &omap_manager, @@ -1472,14 +1465,14 @@ SeaStore::tm_ret SeaStore::_xattr_rmattr( LOG_PREFIX(SeaStore::_xattr_rmattr); DEBUGT("onode={}", *ctx.transaction, *onode); auto xattr_root = onode->get_layout().xattr_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())); + onode->get_metadata_hint(device->get_block_size())); if (xattr_root.is_null()) { return seastar::now(); } else { return seastar::do_with( BtreeOMapManager(*transaction_manager), onode->get_layout().xattr_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())), + onode->get_metadata_hint(device->get_block_size())), std::move(name), [&ctx, &onode](auto &omap_manager, auto &xattr_root, auto &name) { return omap_manager.omap_rm_key(xattr_root, *ctx.transaction, name) @@ -1514,14 +1507,14 @@ SeaStore::tm_ret SeaStore::_xattr_clear( LOG_PREFIX(SeaStore::_xattr_clear); DEBUGT("onode={}", *ctx.transaction, *onode); auto xattr_root = onode->get_layout().xattr_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())); + onode->get_metadata_hint(device->get_block_size())); if (xattr_root.is_null()) { return seastar::now(); } else { return seastar::do_with( BtreeOMapManager(*transaction_manager), onode->get_layout().xattr_root.get( - onode->get_metadata_hint(segment_manager->get_block_size())), + onode->get_metadata_hint(device->get_block_size())), [&ctx, &onode](auto &omap_manager, auto &xattr_root) { return omap_manager.omap_clear(xattr_root, *ctx.transaction) .si_then([&] { @@ -1650,21 +1643,21 @@ seastar::future> SeaStore::read_meta(const std::str uuid_d SeaStore::get_fsid() const { - return segment_manager->get_meta().seastore_id; + return device->get_meta().seastore_id; } seastar::future> make_seastore( const std::string &device, const ConfigValues &config) { - return SegmentManager::get_segment_manager( + return Device::make_device( device - ).then([&device](auto sm) { - auto tm = make_transaction_manager(*sm, false /* detailed */); + ).then([&device](DeviceRef device_obj) { + auto tm = make_transaction_manager(*device_obj, false /* detailed */); auto cm = std::make_unique(*tm); return std::make_unique( device, - std::move(sm), + std::move(device_obj), std::move(tm), std::move(cm), std::make_unique(*tm)); diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index d75b07a06defb..87ac307fc152d 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -19,6 +19,7 @@ #include "crimson/os/futurized_collection.h" #include "crimson/os/futurized_store.h" +#include "crimson/os/seastore/device.h" #include "crimson/os/seastore/transaction.h" #include "crimson/os/seastore/onode_manager.h" #include "crimson/os/seastore/omap_manager.h" @@ -66,13 +67,13 @@ public: SeaStore( const std::string& root, MDStoreRef mdstore, - SegmentManagerRef sm, + DeviceRef device, TransactionManagerRef tm, CollectionManagerRef cm, OnodeManagerRef om); SeaStore( const std::string& root, - SegmentManagerRef sm, + DeviceRef device, TransactionManagerRef tm, CollectionManagerRef cm, OnodeManagerRef om); @@ -312,8 +313,8 @@ private: std::string root; MDStoreRef mdstore; - SegmentManagerRef segment_manager; - std::vector secondaries; + DeviceRef device; + std::vector secondaries; TransactionManagerRef transaction_manager; CollectionManagerRef collection_manager; OnodeManagerRef onode_manager; diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc index 5e9365b38c0ae..d98a75dea747d 100644 --- a/src/crimson/os/seastore/seastore_types.cc +++ b/src/crimson/os/seastore/seastore_types.cc @@ -596,7 +596,8 @@ std::ostream& operator<<(std::ostream& out, placement_hint_t h) bool can_delay_allocation(device_type_t type) { // Some types of device may not support delayed allocation, for example PMEM. - return type <= device_type_t::RANDOM_BLOCK; + return (type >= device_type_t::NONE && + type <= device_type_t::RANDOM_BLOCK); } device_type_t string_to_device_type(std::string type) { diff --git a/src/crimson/os/seastore/segment_manager.cc b/src/crimson/os/seastore/segment_manager.cc index 2ed74625242f3..4d445313a01f3 100644 --- a/src/crimson/os/seastore/segment_manager.cc +++ b/src/crimson/os/seastore/segment_manager.cc @@ -22,15 +22,6 @@ seastar::logger &logger(){ namespace crimson::os::seastore { -std::ostream& operator<<(std::ostream& out, const device_spec_t& ds) -{ - return out << "device_spec(" - << "magic=" << ds.magic - << ", device_type=" << ds.dtype - << ", device_id=" << std::to_string(ds.id) - << ")"; -} - std::ostream& operator<<(std::ostream& out, const block_sm_superblock_t& sb) { out << "superblock(" @@ -52,21 +43,6 @@ std::ostream& operator<<(std::ostream& out, const block_sm_superblock_t& sb) return out << "))"; } -std::ostream& operator<<(std::ostream& out, const segment_manager_config_t& conf) -{ - out << "sm_config_t(" - << "major_dev=" << conf.major_dev - << ", magic=" << conf.magic - << ", device_type=" << conf.dtype - << ", device_id=" << std::to_string(conf.device_id) - << ", meta=" << conf.meta - << ", secondary("; - for (const auto& [k, v] : conf.secondary_devices) { - out << std::to_string(k) << ": " << v; - } - return out << "))"; -} - std::ostream& operator<<(std::ostream &out, Segment::segment_state_t s) { using state_t = Segment::segment_state_t; diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h index 22a5aeae05686..eb4cc6ccab27e 100644 --- a/src/crimson/os/seastore/segment_manager.h +++ b/src/crimson/os/seastore/segment_manager.h @@ -20,26 +20,6 @@ namespace crimson::os::seastore { -using magic_t = uint64_t; - -struct device_spec_t{ - magic_t magic; - device_type_t dtype; - device_id_t id; - DENC(device_spec_t, v, p) { - DENC_START(1, 1, p); - denc(v.magic, p); - denc(v.dtype, p); - denc(v.id, p); - DENC_FINISH(p); - } -}; - -std::ostream& operator<<(std::ostream&, const device_spec_t&); - -using secondary_device_set_t = - std::map; - struct block_sm_superblock_t { size_t size = 0; size_t segment_size = 0; @@ -103,17 +83,6 @@ struct block_sm_superblock_t { std::ostream& operator<<(std::ostream&, const block_sm_superblock_t&); -struct segment_manager_config_t { - bool major_dev = false; - magic_t magic = 0; - device_type_t dtype = device_type_t::NONE; - device_id_t device_id = DEVICE_ID_NULL; - seastore_meta_t meta; - secondary_device_set_t secondary_devices; -}; - -std::ostream& operator<<(std::ostream&, const segment_manager_config_t&); - class Segment : public boost::intrusive_ref_counter< Segment, boost::thread_unsafe_counter>{ @@ -182,23 +151,9 @@ using SegmentManagerRef = std::unique_ptr; class SegmentManager : public Device { public: - using access_ertr = crimson::errorator< - crimson::ct_error::input_output_error, - crimson::ct_error::permission_denied, - crimson::ct_error::enoent>; - - using mount_ertr = access_ertr; - using mount_ret = access_ertr::future<>; - virtual mount_ret mount() = 0; - - using close_ertr = crimson::errorator< - crimson::ct_error::input_output_error - >; - virtual close_ertr::future<> close() = 0; - - using mkfs_ertr = access_ertr; - using mkfs_ret = mkfs_ertr::future<>; - virtual mkfs_ret mkfs(segment_manager_config_t meta) = 0; + device_type_t get_device_type() const final { + return device_type_t::SEGMENTED; + } using open_ertr = crimson::errorator< crimson::ct_error::input_output_error, @@ -213,7 +168,6 @@ public: virtual release_ertr::future<> release(segment_id_t id) = 0; /* Methods for discovering device geometry, segmentid set, etc */ - virtual size_t get_size() const = 0; virtual seastore_off_t get_segment_size() const = 0; virtual device_segment_id_t get_num_segments() const { ceph_assert(get_size() % get_segment_size() == 0); @@ -224,13 +178,6 @@ public: ceph::encoded_sizeof_bounded(), (size_t)get_block_size()); } - virtual const seastore_meta_t &get_meta() const = 0; - - virtual secondary_device_set_t& get_secondary_devices() = 0; - - virtual device_spec_t get_device_spec() const = 0; - - virtual magic_t get_magic() const = 0; virtual ~SegmentManager() {} @@ -239,9 +186,6 @@ public: } -WRITE_CLASS_DENC( - crimson::os::seastore::device_spec_t -) WRITE_CLASS_DENC( crimson::os::seastore::block_sm_superblock_t ) diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index c3814b3cd959a..65403cdcd2f59 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -196,7 +196,7 @@ SegmentStateTracker::read_in( static block_sm_superblock_t make_superblock( device_id_t device_id, - segment_manager_config_t sm_config, + device_config_t sm_config, const seastar::stat_data &data) { LOG_PREFIX(block_make_superblock); @@ -517,7 +517,7 @@ BlockSegmentManager::mount_ret BlockSegmentManager::mount() } BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs( - segment_manager_config_t sm_config) + device_config_t sm_config) { LOG_PREFIX(BlockSegmentManager::mkfs); set_device_id(sm_config.device_id); diff --git a/src/crimson/os/seastore/segment_manager/block.h b/src/crimson/os/seastore/segment_manager/block.h index 8008c4bd4e32a..363c7c5f23de0 100644 --- a/src/crimson/os/seastore/segment_manager/block.h +++ b/src/crimson/os/seastore/segment_manager/block.h @@ -112,7 +112,7 @@ class BlockSegmentManager final : public SegmentManager { public: mount_ret mount() final; - mkfs_ret mkfs(segment_manager_config_t) final; + mkfs_ret mkfs(device_config_t) final; close_ertr::future<> close(); @@ -154,12 +154,6 @@ public: ceph::bufferlist bl, bool ignore_check=false); - device_spec_t get_device_spec() const final { - return {superblock.magic, - superblock.dtype, - get_device_id()}; - } - magic_t get_magic() const final { return superblock.magic; } diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.h b/src/crimson/os/seastore/segment_manager/ephemeral.h index d9c786815ffa3..a57c87fc24b98 100644 --- a/src/crimson/os/seastore/segment_manager/ephemeral.h +++ b/src/crimson/os/seastore/segment_manager/ephemeral.h @@ -21,7 +21,6 @@ struct ephemeral_config_t { size_t block_size = 0; size_t segment_size = 0; magic_t magic = 0; - device_type_t dtype = device_type_t::NONE; device_id_t id = 0; }; @@ -30,7 +29,6 @@ constexpr ephemeral_config_t DEFAULT_TEST_EPHEMERAL = { 4 << 10, 8 << 20, 0xabcd, - device_type_t::SEGMENTED, 0 }; @@ -99,7 +97,7 @@ public: return mount_ertr::now(); } - mkfs_ret mkfs(segment_manager_config_t) { + mkfs_ret mkfs(device_config_t) { return mkfs_ertr::now(); } @@ -131,10 +129,6 @@ public: return sec_device_set; } - device_spec_t get_device_spec() const final { - return {config.magic, config.dtype, config.id}; - } - magic_t get_magic() const final { return config.magic; } diff --git a/src/crimson/os/seastore/segment_manager/zns.cc b/src/crimson/os/seastore/segment_manager/zns.cc index ec8ac278c9dbb..0c59f2e1d683a 100644 --- a/src/crimson/os/seastore/segment_manager/zns.cc +++ b/src/crimson/os/seastore/segment_manager/zns.cc @@ -303,7 +303,7 @@ ZNSSegmentManager::mount_ret ZNSSegmentManager::mount() } ZNSSegmentManager::mkfs_ret ZNSSegmentManager::mkfs( - segment_manager_config_t config) + device_config_t config) { logger().error("ZNSSegmentManager::mkfs: starting"); return seastar::do_with( @@ -544,15 +544,6 @@ secondary_device_set_t& ZNSSegmentManager::get_secondary_devices() return metadata.secondary_devices; }; -device_spec_t ZNSSegmentManager::get_device_spec() const -{ - auto spec = device_spec_t(); - spec.magic = metadata.magic; - spec.dtype = metadata.dtype; - spec.id = metadata.device_id; - return spec; -}; - magic_t ZNSSegmentManager::get_magic() const { return metadata.magic; diff --git a/src/crimson/os/seastore/segment_manager/zns.h b/src/crimson/os/seastore/segment_manager/zns.h index 064711187c463..289e730605479 100644 --- a/src/crimson/os/seastore/segment_manager/zns.h +++ b/src/crimson/os/seastore/segment_manager/zns.h @@ -84,7 +84,7 @@ namespace crimson::os::seastore::segment_manager::zns { class ZNSSegmentManager final : public SegmentManager{ public: mount_ret mount() final; - mkfs_ret mkfs(segment_manager_config_t meta) final; + mkfs_ret mkfs(device_config_t meta) final; open_ertr::future open(segment_id_t id) final; close_ertr::future<> close() final; @@ -115,8 +115,6 @@ namespace crimson::os::seastore::segment_manager::zns { secondary_device_set_t& get_secondary_devices() final; - device_spec_t get_device_spec() const final; - magic_t get_magic() const final; ZNSSegmentManager(const std::string &path) : device_path(path) {} diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index e59f9332315ea..b922aedb55234 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -542,7 +542,7 @@ TransactionManager::get_extent_if_live_ret TransactionManager::get_extent_if_liv TransactionManager::~TransactionManager() {} TransactionManagerRef make_transaction_manager( - SegmentManager& sm, + Device &device, bool detailed) { auto scanner = std::make_unique(); @@ -551,7 +551,10 @@ TransactionManagerRef make_transaction_manager( SegmentCleaner::config_t::get_default(), std::move(scanner), detailed); - auto journal = journal::make_segmented(sm, scanner_ref, *segment_cleaner); + ceph_assert(device.get_device_type() == device_type_t::SEGMENTED); + auto sm = dynamic_cast(&device); + ceph_assert(sm != nullptr); + auto journal = journal::make_segmented(*sm, scanner_ref, *segment_cleaner); auto epm = std::make_unique(); auto cache = std::make_unique(*epm); auto lba_manager = lba_manager::create_lba_manager(*cache); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 5e0b6916beb90..cc54ba2fb1515 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -27,6 +27,7 @@ #include "crimson/os/seastore/lba_manager.h" #include "crimson/os/seastore/journal.h" #include "crimson/os/seastore/extent_placement_manager.h" +#include "crimson/os/seastore/device.h" namespace crimson::os::seastore { class Journal; @@ -532,18 +533,22 @@ public: return segment_cleaner->stat(); } - void add_segment_manager(SegmentManager* sm, bool is_primary) { - LOG_PREFIX(TransactionManager::add_segment_manager); - SUBDEBUG(seastore_tm, "adding segment manager {}, is_primary={}", - sm->get_device_id(), is_primary); - scanner.add_segment_manager(sm); - epm->add_device(sm, is_primary); + void add_device(Device* dev, bool is_primary) { + LOG_PREFIX(TransactionManager::add_device); + SUBDEBUG(seastore_tm, "adding device {}, is_primary={}", + dev->get_device_id(), is_primary); + epm->add_device(dev, is_primary); + + ceph_assert(dev->get_device_type() == device_type_t::SEGMENTED); + auto sm = dynamic_cast(dev); + ceph_assert(sm != nullptr); epm->add_allocator( - device_type_t::SEGMENTED, + dev->get_device_type(), std::make_unique( *segment_cleaner, *sm, segment_cleaner->get_ool_segment_seq_allocator())); + scanner.add_segment_manager(sm); } ~TransactionManager(); @@ -576,7 +581,7 @@ public: using TransactionManagerRef = std::unique_ptr; TransactionManagerRef make_transaction_manager( - SegmentManager& sm, + Device &device, bool detailed); } diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc index 54f6912349aff..76f4825ec8c79 100644 --- a/src/crimson/tools/store_nbd/tm_driver.cc +++ b/src/crimson/tools/store_nbd/tm_driver.cc @@ -3,12 +3,9 @@ #include "tm_driver.h" -#include "crimson/os/seastore/segment_manager/block.h" - using namespace crimson; using namespace crimson::os; using namespace crimson::os::seastore; -using namespace crimson::os::seastore::segment_manager::block; namespace { seastar::logger& logger() { @@ -21,8 +18,8 @@ seastar::future<> TMDriver::write( bufferptr ptr) { logger().debug("Writing offset {}", offset); - assert(offset % segment_manager->get_block_size() == 0); - assert((ptr.length() % (size_t)segment_manager->get_block_size()) == 0); + assert(offset % device->get_block_size() == 0); + assert((ptr.length() % (size_t)device->get_block_size()) == 0); return seastar::do_with(ptr, [this, offset](auto& ptr) { return repeat_eagain([this, offset, &ptr] { return tm->with_transaction_intr( @@ -96,8 +93,8 @@ seastar::future TMDriver::read( size_t size) { logger().debug("Reading offset {}", offset); - assert(offset % segment_manager->get_block_size() == 0); - assert(size % (size_t)segment_manager->get_block_size() == 0); + assert(offset % device->get_block_size() == 0); + assert(size % (size_t)device->get_block_size() == 0); auto blptrret = std::make_unique(); auto &blret = *blptrret; return repeat_eagain([=, &blret] { @@ -134,8 +131,8 @@ seastar::future TMDriver::read( void TMDriver::init() { - tm = make_transaction_manager(*segment_manager, false /* detailed */); - tm->add_segment_manager(segment_manager.get(), true); + tm = make_transaction_manager(*device, false /* detailed */); + tm->add_device(device.get(), true); } void TMDriver::clear() @@ -145,29 +142,29 @@ void TMDriver::clear() size_t TMDriver::get_size() const { - return segment_manager->get_size() * .5; + return device->get_size() * .5; } seastar::future<> TMDriver::mkfs() { assert(config.path); - segment_manager = std::make_unique< - segment_manager::block::BlockSegmentManager - >(*config.path); logger().debug("mkfs"); - seastore_meta_t meta; - meta.seastore_id.generate_random(); - return segment_manager->mkfs( - segment_manager_config_t{ - true, - (magic_t)std::rand(), - device_type_t::SEGMENTED, - 0, - meta, - secondary_device_set_t()} - ).safe_then([this] { - logger().debug(""); - return segment_manager->mount(); + return Device::make_device(*config.path + ).then([this](DeviceRef dev) { + device = std::move(dev); + seastore_meta_t meta; + meta.seastore_id.generate_random(); + return device->mkfs( + device_config_t{ + true, + (magic_t)std::rand(), + device_type_t::SEGMENTED, + 0, + meta, + secondary_device_set_t()}); + }).safe_then([this] { + logger().debug("device mkfs done"); + return device->mount(); }).safe_then([this] { init(); logger().debug("tm mkfs"); @@ -177,10 +174,10 @@ seastar::future<> TMDriver::mkfs() return tm->close(); }).safe_then([this] { logger().debug("sm close"); - return segment_manager->close(); + return device->close(); }).safe_then([this] { clear(); - segment_manager.reset(); + device.reset(); logger().debug("mkfs complete"); return TransactionManager::mkfs_ertr::now(); }).handle_error( @@ -194,10 +191,10 @@ seastar::future<> TMDriver::mount() { return (config.mkfs ? mkfs() : seastar::now() ).then([this] { - segment_manager = std::make_unique< - segment_manager::block::BlockSegmentManager - >(*config.path); - return segment_manager->mount(); + return Device::make_device(*config.path); + }).then([this](DeviceRef dev) { + device = std::move(dev); + return device->mount(); }).safe_then([this] { init(); return tm->mount(); @@ -212,7 +209,7 @@ seastar::future<> TMDriver::close() { return tm->close().safe_then([this] { clear(); - return segment_manager->close(); + return device->close(); }).handle_error( crimson::ct_error::assert_all{ "Invalid errror during TMDriver::close" diff --git a/src/crimson/tools/store_nbd/tm_driver.h b/src/crimson/tools/store_nbd/tm_driver.h index f6d7c9b8d5cbd..26a200fa3fc2b 100644 --- a/src/crimson/tools/store_nbd/tm_driver.h +++ b/src/crimson/tools/store_nbd/tm_driver.h @@ -5,7 +5,7 @@ #include "crimson/os/seastore/cache.h" #include "crimson/os/seastore/segment_cleaner.h" -#include "crimson/os/seastore/segment_manager.h" +#include "crimson/os/seastore/device.h" #include "crimson/os/seastore/transaction_manager.h" #include "test/crimson/seastore/test_block.h" @@ -35,8 +35,8 @@ public: private: const config_t config; - using SegmentManagerRef = crimson::os::seastore::SegmentManagerRef; - SegmentManagerRef segment_manager; + using DeviceRef = crimson::os::seastore::DeviceRef; + DeviceRef device; using TransactionManager = crimson::os::seastore::TransactionManager; using TransactionManagerRef = crimson::os::seastore::TransactionManagerRef; diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc index 3ff93fb4f85d2..892e5f780e953 100644 --- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc +++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc @@ -84,7 +84,7 @@ struct fltree_onode_manager_test_t virtual FuturizedStore::mkfs_ertr::future<> _mkfs() final { return TMTestState::_mkfs( ).safe_then([this] { - tm->add_segment_manager(segment_manager.get(), true); + tm->add_device(segment_manager.get(), true); return tm->mount( ).safe_then([this] { return repeat_eagain([this] { diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 1dd72325f6727..e987cd44fcc53 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -93,7 +93,7 @@ protected: virtual void _init() override { tm = make_transaction_manager(*segment_manager, true); - tm->add_segment_manager(segment_manager.get(), true); + tm->add_device(segment_manager.get(), true); segment_cleaner = tm->get_segment_cleaner(); lba_manager = tm->get_lba_manager(); } @@ -182,7 +182,7 @@ public: return mount_ertr::now(); // we handle this above } - mkfs_ret mkfs(segment_manager_config_t c) final { + mkfs_ret mkfs(device_config_t c) final { return mkfs_ertr::now(); // we handle this above } @@ -194,10 +194,6 @@ public: return sm.get_secondary_devices(); } - device_spec_t get_device_spec() const final { - return sm.get_device_spec(); - } - magic_t get_magic() const final { return sm.get_magic(); } -- 2.39.5