From e15e32468ba3c27350e9009e7ae042111dda8cce Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Tue, 4 May 2021 17:16:57 +0000 Subject: [PATCH] crimson/tools/store-nbd: break into multiple files Signed-off-by: Samuel Just --- src/crimson/tools/CMakeLists.txt | 4 +- src/crimson/tools/store_nbd/block_driver.cc | 16 + src/crimson/tools/store_nbd/block_driver.h | 74 +++++ .../tools/{ => store_nbd}/store-nbd.cc | 301 +----------------- src/crimson/tools/store_nbd/tm_driver.cc | 228 +++++++++++++ src/crimson/tools/store_nbd/tm_driver.h | 57 ++++ 6 files changed, 381 insertions(+), 299 deletions(-) create mode 100644 src/crimson/tools/store_nbd/block_driver.cc create mode 100644 src/crimson/tools/store_nbd/block_driver.h rename src/crimson/tools/{ => store_nbd}/store-nbd.cc (57%) create mode 100644 src/crimson/tools/store_nbd/tm_driver.cc create mode 100644 src/crimson/tools/store_nbd/tm_driver.h diff --git a/src/crimson/tools/CMakeLists.txt b/src/crimson/tools/CMakeLists.txt index 1a59a9a1170..f044e1d3e7d 100644 --- a/src/crimson/tools/CMakeLists.txt +++ b/src/crimson/tools/CMakeLists.txt @@ -1,5 +1,7 @@ add_executable(crimson-store-nbd - store-nbd.cc + store_nbd/store-nbd.cc + store_nbd/tm_driver.cc + store_nbd/block_driver.cc ) target_link_libraries(crimson-store-nbd crimson-seastore) diff --git a/src/crimson/tools/store_nbd/block_driver.cc b/src/crimson/tools/store_nbd/block_driver.cc new file mode 100644 index 00000000000..f4678dbd40c --- /dev/null +++ b/src/crimson/tools/store_nbd/block_driver.cc @@ -0,0 +1,16 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "block_driver.h" + +#include "tm_driver.h" + +BlockDriverRef get_backend(BlockDriver::config_t config) +{ + if (config.type == "transaction_manager") { + return std::make_unique(config); + } else { + ceph_assert(0 == "invalid option"); + return BlockDriverRef(); + } +} diff --git a/src/crimson/tools/store_nbd/block_driver.h b/src/crimson/tools/store_nbd/block_driver.h new file mode 100644 index 00000000000..c92475c07d0 --- /dev/null +++ b/src/crimson/tools/store_nbd/block_driver.h @@ -0,0 +1,74 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include +#include + +#include + +#include +#include + +#include "include/buffer.h" + +/** + * BlockDriver + * + * Simple interface to enable throughput test to compare raw disk to + * transaction_manager, etc + */ +class BlockDriver { +public: + struct config_t { + std::string type; + bool mkfs = false; + std::optional path; + + void populate_options( + boost::program_options::options_description &desc) + { + namespace po = boost::program_options; + desc.add_options() + ("type", + po::value() + ->default_value("transaction_manager") + ->notifier([this](auto s) { type = s; }), + "Backend to use, options are transaction_manager" + ) + ("device-path", + po::value() + ->required() + ->notifier([this](auto s) { path = s; }), + "Path to device for backend" + ) + ("mkfs", + po::value() + ->default_value(false) + ->notifier([this](auto s) { mkfs = s; }), + "Do mkfs first" + ); + } + }; + + virtual ceph::bufferptr get_buffer(size_t size) = 0; + + virtual seastar::future<> write( + off_t offset, + ceph::bufferptr ptr) = 0; + + virtual seastar::future read( + off_t offset, + size_t size) = 0; + + virtual size_t get_size() const = 0; + + virtual seastar::future<> mount() = 0; + virtual seastar::future<> close() = 0; + + virtual ~BlockDriver() {} +}; +using BlockDriverRef = std::unique_ptr; + +BlockDriverRef get_backend(BlockDriver::config_t config); diff --git a/src/crimson/tools/store-nbd.cc b/src/crimson/tools/store_nbd/store-nbd.cc similarity index 57% rename from src/crimson/tools/store-nbd.cc rename to src/crimson/tools/store_nbd/store-nbd.cc index 0d4e0e96d29..fa36d03bad7 100644 --- a/src/crimson/tools/store-nbd.cc +++ b/src/crimson/tools/store_nbd/store-nbd.cc @@ -38,25 +38,17 @@ #include #include +#include "crimson/common/log.h" #include "crimson/common/config_proxy.h" #include "crimson/common/perf_counters_collection.h" -#include "crimson/os/seastore/cache.h" -#include "crimson/os/seastore/segment_cleaner.h" -#include "crimson/os/seastore/segment_manager.h" -#include "crimson/os/seastore/segment_manager/block.h" -#include "crimson/os/seastore/transaction_manager.h" - #include "test/crimson/seastar_runner.h" -#include "test/crimson/seastore/test_block.h" + +#include "block_driver.h" namespace po = boost::program_options; using namespace ceph; -using namespace crimson; -using namespace crimson::os; -using namespace crimson::os::seastore; -using namespace crimson::os::seastore::segment_manager::block; namespace { seastar::logger& logger() { @@ -64,65 +56,6 @@ namespace { } } -/** - * BlockDriver - * - * Simple interface to enable throughput test to compare raw disk to - * transaction_manager, etc - */ -class BlockDriver { -public: - struct config_t { - std::string type; - bool mkfs = false; - std::optional path; - - void populate_options( - po::options_description &desc) - { - desc.add_options() - ("type", - po::value() - ->default_value("transaction_manager") - ->notifier([this](auto s) { type = s; }), - "Backend to use, options are transaction_manager" - ) - ("device-path", - po::value() - ->required() - ->notifier([this](auto s) { path = s; }), - "Path to device for backend" - ) - ("mkfs", - po::value() - ->default_value(false) - ->notifier([this](auto s) { mkfs = s; }), - "Do mkfs first" - ); - } - }; - - virtual bufferptr get_buffer(size_t size) = 0; - - virtual seastar::future<> write( - off_t offset, - bufferptr ptr) = 0; - - virtual seastar::future read( - off_t offset, - size_t size) = 0; - - virtual size_t get_size() const = 0; - - virtual seastar::future<> mount() = 0; - virtual seastar::future<> close() = 0; - - virtual ~BlockDriver() {} -}; -using BlockDriverRef = std::unique_ptr; - -BlockDriverRef get_backend(BlockDriver::config_t config); - struct request_context_t { uint32_t magic = 0; uint32_t type = 0; @@ -504,231 +437,3 @@ seastar::future<> NBDHandler::run() }); }); } - -class TMDriver final : public BlockDriver { - const config_t config; - std::unique_ptr segment_manager; - std::unique_ptr tm; - -public: - TMDriver(config_t config) : config(config) {} - ~TMDriver() final {} - - bufferptr get_buffer(size_t size) final { - return ceph::buffer::create_page_aligned(size); - } - - seastar::future<> write( - off_t offset, - bufferptr ptr) final { - logger().debug("Writing offset {}", offset); - assert(offset % segment_manager->get_block_size() == 0); - assert((ptr.length() % (size_t)segment_manager->get_block_size()) == 0); - return repeat_eagain([this, offset, ptr=std::move(ptr)] { - return seastar::do_with( - tm->create_transaction(), - ptr, - [this, offset](auto &t, auto &ptr) mutable { - return tm->dec_ref( - *t, - offset - ).safe_then([](auto){}).handle_error( - crimson::ct_error::enoent::handle([](auto) { return seastar::now(); }), - crimson::ct_error::pass_further_all{} - ).safe_then([=, &t, &ptr] { - logger().debug("dec_ref complete"); - return tm->alloc_extent( - *t, - offset, - ptr.length()); - }).safe_then([=, &t, &ptr](auto ext) mutable { - assert(ext->get_laddr() == (size_t)offset); - assert(ext->get_bptr().length() == ptr.length()); - ext->get_bptr().swap(ptr); - logger().debug("submitting transaction"); - return tm->submit_transaction(std::move(t)); - }); - }); - }).handle_error( - crimson::ct_error::assert_all{"store-nbd write"} - ); - } - - auto read_extents( - Transaction &t, - laddr_t offset, - extent_len_t length) { - return seastar::do_with( - lba_pin_list_t(), - lextent_list_t(), - [this, &t, offset, length](auto &pins, auto &ret) { - return tm->get_pins( - t, offset, length - ).safe_then([this, &t, &pins, &ret](auto _pins) { - _pins.swap(pins); - logger().debug("read_extents: mappings {}", pins); - return crimson::do_for_each( - pins.begin(), - pins.end(), - [this, &t, &ret](auto &&pin) { - logger().debug( - "read_extents: get_extent {}~{}", - pin->get_paddr(), - pin->get_length()); - return tm->pin_to_extent( - t, - std::move(pin) - ).safe_then([this, &ret](auto ref) mutable { - ret.push_back(std::make_pair(ref->get_laddr(), ref)); - logger().debug( - "read_extents: got extent {}", - *ref); - return seastar::now(); - }); - }).safe_then([&ret] { - return std::move(ret); - }); - }); - }); - } - - seastar::future read( - off_t offset, - size_t size) final { - logger().debug("Reading offset {}", offset); - assert(offset % segment_manager->get_block_size() == 0); - assert(size % (size_t)segment_manager->get_block_size() == 0); - auto blptrret = std::make_unique(); - auto &blret = *blptrret; - return repeat_eagain([=, &blret] { - return seastar::do_with( - tm->create_transaction(), - [=, &blret](auto &t) { - return read_extents(*t, offset, size - ).safe_then([=, &blret](auto ext_list) mutable { - size_t cur = offset; - for (auto &i: ext_list) { - if (cur != i.first) { - assert(cur < i.first); - blret.append_zero(i.first - cur); - cur = i.first; - } - blret.append(i.second->get_bptr()); - cur += i.second->get_bptr().length(); - } - if (blret.length() != size) { - assert(blret.length() < size); - blret.append_zero(size - blret.length()); - } - }); - }); - }).handle_error( - crimson::ct_error::assert_all{"store-nbd read"} - ).then([blptrret=std::move(blptrret)]() mutable { - logger().debug("read complete"); - return std::move(*blptrret); - }); - } - - void init() { - auto segment_cleaner = std::make_unique( - SegmentCleaner::config_t::get_default(), - false /* detailed */); - segment_cleaner->mount(*segment_manager); - auto journal = std::make_unique(*segment_manager); - auto cache = std::make_unique(*segment_manager); - auto lba_manager = lba_manager::create_lba_manager(*segment_manager, *cache); - - journal->set_segment_provider(&*segment_cleaner); - - tm = std::make_unique( - *segment_manager, - std::move(segment_cleaner), - std::move(journal), - std::move(cache), - std::move(lba_manager)); - } - - void clear() { - tm.reset(); - } - - size_t get_size() const final { - return segment_manager->get_size() * .5; - } - - seastar::future<> mkfs() { - assert(config.path); - segment_manager = std::make_unique< - segment_manager::block::BlockSegmentManager - >(*config.path); - logger().debug("mkfs"); - seastore_meta_t meta; - meta.seastore_id.generate_random(); - return segment_manager->mkfs( - std::move(meta) - ).safe_then([this] { - logger().debug(""); - return segment_manager->mount(); - }).safe_then([this] { - init(); - logger().debug("tm mkfs"); - return tm->mkfs(); - }).safe_then([this] { - logger().debug("tm close"); - return tm->close(); - }).safe_then([this] { - logger().debug("sm close"); - return segment_manager->close(); - }).safe_then([this] { - clear(); - logger().debug("mkfs complete"); - return TransactionManager::mkfs_ertr::now(); - }).handle_error( - crimson::ct_error::assert_all{ - "Invalid errror during TMDriver::mkfs" - } - ); - } - - seastar::future<> mount() final { - return (config.mkfs ? mkfs() : seastar::now() - ).then([this] { - segment_manager = std::make_unique< - segment_manager::block::BlockSegmentManager - >(*config.path); - return segment_manager->mount(); - }).safe_then([this] { - init(); - return tm->mount(); - }).handle_error( - crimson::ct_error::assert_all{ - "Invalid errror during TMDriver::mount" - } - ); - }; - - seastar::future<> close() final { - return segment_manager->close( - ).safe_then([this] { - return tm->close(); - }).safe_then([this] { - clear(); - return seastar::now(); - }).handle_error( - crimson::ct_error::assert_all{ - "Invalid errror during TMDriver::close" - } - ); - } -}; - -BlockDriverRef get_backend(BlockDriver::config_t config) -{ - if (config.type == "transaction_manager") { - return std::make_unique(config); - } else { - ceph_assert(0 == "invalid option"); - return BlockDriverRef(); - } -} diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc new file mode 100644 index 00000000000..b45bd0d52b8 --- /dev/null +++ b/src/crimson/tools/store_nbd/tm_driver.cc @@ -0,0 +1,228 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tm_driver.h" + +using namespace crimson; +using namespace crimson::os; +using namespace crimson::os::seastore; +using namespace crimson::os::seastore::segment_manager::block; + +namespace { + seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_test); + } +} + +seastar::future<> TMDriver::write( + off_t offset, + bufferptr ptr) +{ + logger().debug("Writing offset {}", offset); + assert(offset % segment_manager->get_block_size() == 0); + assert((ptr.length() % (size_t)segment_manager->get_block_size()) == 0); + return repeat_eagain([this, offset, ptr=std::move(ptr)] { + return seastar::do_with( + tm->create_transaction(), + ptr, + [this, offset](auto &t, auto &ptr) mutable { + return tm->dec_ref( + *t, + offset + ).safe_then([](auto){}).handle_error( + crimson::ct_error::enoent::handle([](auto) { return seastar::now(); }), + crimson::ct_error::pass_further_all{} + ).safe_then([=, &t, &ptr] { + logger().debug("dec_ref complete"); + return tm->alloc_extent( + *t, + offset, + ptr.length()); + }).safe_then([=, &t, &ptr](auto ext) mutable { + assert(ext->get_laddr() == (size_t)offset); + assert(ext->get_bptr().length() == ptr.length()); + ext->get_bptr().swap(ptr); + logger().debug("submitting transaction"); + return tm->submit_transaction(std::move(t)); + }); + }); + }).handle_error( + crimson::ct_error::assert_all{"store-nbd write"} + ); +} + +TMDriver::read_extents_ret TMDriver::read_extents( + Transaction &t, + laddr_t offset, + extent_len_t length) +{ + return seastar::do_with( + lba_pin_list_t(), + lextent_list_t(), + [this, &t, offset, length](auto &pins, auto &ret) { + return tm->get_pins( + t, offset, length + ).safe_then([this, &t, &pins, &ret](auto _pins) { + _pins.swap(pins); + logger().debug("read_extents: mappings {}", pins); + return crimson::do_for_each( + pins.begin(), + pins.end(), + [this, &t, &ret](auto &&pin) { + logger().debug( + "read_extents: get_extent {}~{}", + pin->get_paddr(), + pin->get_length()); + return tm->pin_to_extent( + t, + std::move(pin) + ).safe_then([this, &ret](auto ref) mutable { + ret.push_back(std::make_pair(ref->get_laddr(), ref)); + logger().debug( + "read_extents: got extent {}", + *ref); + return seastar::now(); + }); + }).safe_then([&ret] { + return std::move(ret); + }); + }); + }); +} + +seastar::future TMDriver::read( + off_t offset, + size_t size) +{ + logger().debug("Reading offset {}", offset); + assert(offset % segment_manager->get_block_size() == 0); + assert(size % (size_t)segment_manager->get_block_size() == 0); + auto blptrret = std::make_unique(); + auto &blret = *blptrret; + return repeat_eagain([=, &blret] { + return seastar::do_with( + tm->create_transaction(), + [=, &blret](auto &t) { + return read_extents(*t, offset, size + ).safe_then([=, &blret](auto ext_list) mutable { + size_t cur = offset; + for (auto &i: ext_list) { + if (cur != i.first) { + assert(cur < i.first); + blret.append_zero(i.first - cur); + cur = i.first; + } + blret.append(i.second->get_bptr()); + cur += i.second->get_bptr().length(); + } + if (blret.length() != size) { + assert(blret.length() < size); + blret.append_zero(size - blret.length()); + } + }); + }); + }).handle_error( + crimson::ct_error::assert_all{"store-nbd read"} + ).then([blptrret=std::move(blptrret)]() mutable { + logger().debug("read complete"); + return std::move(*blptrret); + }); +} + +void TMDriver::init() +{ + auto segment_cleaner = std::make_unique( + SegmentCleaner::config_t::get_default(), + false /* detailed */); + segment_cleaner->mount(*segment_manager); + auto journal = std::make_unique(*segment_manager); + auto cache = std::make_unique(*segment_manager); + auto lba_manager = lba_manager::create_lba_manager(*segment_manager, *cache); + + journal->set_segment_provider(&*segment_cleaner); + + tm = std::make_unique( + *segment_manager, + std::move(segment_cleaner), + std::move(journal), + std::move(cache), + std::move(lba_manager)); +} + +void TMDriver::clear() +{ + tm.reset(); +} + +size_t TMDriver::get_size() const +{ + return segment_manager->get_size() * .5; +} + +seastar::future<> TMDriver::mkfs() +{ + assert(config.path); + segment_manager = std::make_unique< + segment_manager::block::BlockSegmentManager + >(*config.path); + logger().debug("mkfs"); + seastore_meta_t meta; + meta.seastore_id.generate_random(); + return segment_manager->mkfs( + std::move(meta) + ).safe_then([this] { + logger().debug(""); + return segment_manager->mount(); + }).safe_then([this] { + init(); + logger().debug("tm mkfs"); + return tm->mkfs(); + }).safe_then([this] { + logger().debug("tm close"); + return tm->close(); + }).safe_then([this] { + logger().debug("sm close"); + return segment_manager->close(); + }).safe_then([this] { + clear(); + logger().debug("mkfs complete"); + return TransactionManager::mkfs_ertr::now(); + }).handle_error( + crimson::ct_error::assert_all{ + "Invalid errror during TMDriver::mkfs" + } + ); +} + +seastar::future<> TMDriver::mount() +{ + return (config.mkfs ? mkfs() : seastar::now() + ).then([this] { + segment_manager = std::make_unique< + segment_manager::block::BlockSegmentManager + >(*config.path); + return segment_manager->mount(); + }).safe_then([this] { + init(); + return tm->mount(); + }).handle_error( + crimson::ct_error::assert_all{ + "Invalid errror during TMDriver::mount" + } + ); +}; + +seastar::future<> TMDriver::close() +{ + return segment_manager->close( + ).safe_then([this] { + return tm->close(); + }).safe_then([this] { + clear(); + return seastar::now(); + }).handle_error( + crimson::ct_error::assert_all{ + "Invalid errror during TMDriver::close" + } + ); +} diff --git a/src/crimson/tools/store_nbd/tm_driver.h b/src/crimson/tools/store_nbd/tm_driver.h new file mode 100644 index 00000000000..1643281339f --- /dev/null +++ b/src/crimson/tools/store_nbd/tm_driver.h @@ -0,0 +1,57 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "block_driver.h" + +#include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/segment_cleaner.h" +#include "crimson/os/seastore/segment_manager.h" +#include "crimson/os/seastore/segment_manager/block.h" +#include "crimson/os/seastore/transaction_manager.h" +#include "test/crimson/seastore/test_block.h" + +class TMDriver final : public BlockDriver { +public: + TMDriver(config_t config) : config(config) {} + ~TMDriver() final {} + + bufferptr get_buffer(size_t size) final { + return ceph::buffer::create_page_aligned(size); + } + + seastar::future<> write( + off_t offset, + bufferptr ptr) final; + + seastar::future read( + off_t offset, + size_t size) final; + + size_t get_size() const final; + + seastar::future<> mount() final; + + seastar::future<> close() final; + +private: + const config_t config; + + using BlockSegmentManager = crimson::os::seastore::segment_manager::block::BlockSegmentManager; + std::unique_ptr segment_manager; + + using TransactionManager = crimson::os::seastore::TransactionManager; + std::unique_ptr tm; + + seastar::future<> mkfs(); + void init(); + void clear(); + + using read_extents_ertr = TransactionManager::read_extent_ertr; + using read_extents_ret = read_extents_ertr::future< + crimson::os::seastore::lextent_list_t + >; + read_extents_ret read_extents( + crimson::os::seastore::Transaction &t, + crimson::os::seastore::laddr_t offset, + crimson::os::seastore::extent_len_t length); +}; -- 2.39.5