From b612df65f60a0dc0cd43bd06e507426039e2e985 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Fri, 7 May 2021 18:56:14 -0700 Subject: [PATCH] crimson/tools/store_nbd: add FuturizedStore driver Allows usage of bluestore and seastore. Signed-off-by: Samuel Just --- src/crimson/tools/CMakeLists.txt | 3 +- src/crimson/tools/store_nbd/block_driver.cc | 3 + src/crimson/tools/store_nbd/block_driver.h | 23 ++- src/crimson/tools/store_nbd/fs_driver.cc | 166 ++++++++++++++++++++ src/crimson/tools/store_nbd/fs_driver.h | 49 ++++++ 5 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 src/crimson/tools/store_nbd/fs_driver.cc create mode 100644 src/crimson/tools/store_nbd/fs_driver.h diff --git a/src/crimson/tools/CMakeLists.txt b/src/crimson/tools/CMakeLists.txt index f044e1d3e7dbc..52436c62dc48d 100644 --- a/src/crimson/tools/CMakeLists.txt +++ b/src/crimson/tools/CMakeLists.txt @@ -1,8 +1,9 @@ add_executable(crimson-store-nbd store_nbd/store-nbd.cc store_nbd/tm_driver.cc + store_nbd/fs_driver.cc store_nbd/block_driver.cc ) target_link_libraries(crimson-store-nbd - crimson-seastore) + crimson-os) install(TARGETS crimson-store-nbd DESTINATION bin) diff --git a/src/crimson/tools/store_nbd/block_driver.cc b/src/crimson/tools/store_nbd/block_driver.cc index f4678dbd40c5a..10e77a34bc39d 100644 --- a/src/crimson/tools/store_nbd/block_driver.cc +++ b/src/crimson/tools/store_nbd/block_driver.cc @@ -1,6 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include "fs_driver.h" #include "block_driver.h" #include "tm_driver.h" @@ -9,6 +10,8 @@ BlockDriverRef get_backend(BlockDriver::config_t config) { if (config.type == "transaction_manager") { return std::make_unique(config); + } else if (config.is_futurized_store()) { + return std::make_unique(config); } else { ceph_assert(0 == "invalid option"); return BlockDriverRef(); diff --git a/src/crimson/tools/store_nbd/block_driver.h b/src/crimson/tools/store_nbd/block_driver.h index c92475c07d039..dca74ee43bdcb 100644 --- a/src/crimson/tools/store_nbd/block_driver.h +++ b/src/crimson/tools/store_nbd/block_driver.h @@ -24,8 +24,19 @@ public: struct config_t { std::string type; bool mkfs = false; + unsigned num_collections = 128; + unsigned object_size = 4<<20 /* 4MB, rbd default */; std::optional path; + bool is_futurized_store() const { + return type == "seastore" || type == "bluestore"; + } + + std::string get_fs_type() const { + ceph_assert(is_futurized_store()); + return type; + } + void populate_options( boost::program_options::options_description &desc) { @@ -35,7 +46,7 @@ public: po::value() ->default_value("transaction_manager") ->notifier([this](auto s) { type = s; }), - "Backend to use, options are transaction_manager" + "Backend to use, options are transaction_manager, seastore" ) ("device-path", po::value() @@ -43,6 +54,16 @@ public: ->notifier([this](auto s) { path = s; }), "Path to device for backend" ) + ("num-collections", + po::value() + ->notifier([this](auto s) { num_collections = s; }), + "Number of collections to use for futurized_store backends" + ) + ("object-size", + po::value() + ->notifier([this](auto s) { object_size = s; }), + "Object size to use for futurized_store backends" + ) ("mkfs", po::value() ->default_value(false) diff --git a/src/crimson/tools/store_nbd/fs_driver.cc b/src/crimson/tools/store_nbd/fs_driver.cc new file mode 100644 index 0000000000000..76db2c2708264 --- /dev/null +++ b/src/crimson/tools/store_nbd/fs_driver.cc @@ -0,0 +1,166 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include + +#include "os/Transaction.h" +#include "fs_driver.h" + +using namespace crimson; +using namespace crimson::os; + +coll_t get_coll(unsigned num) { + return coll_t(spg_t(pg_t(0, num))); +} + +FSDriver::offset_mapping_t FSDriver::map_offset(off_t offset) +{ + uint32_t objid = offset / config.object_size; + uint32_t collid = objid % config.num_collections; + return offset_mapping_t{ + collections[collid], + ghobject_t( + shard_id_t::NO_SHARD, + 0, + (objid << 16) | collid, + "", + "", + 0, + ghobject_t::NO_GEN), + offset % config.object_size + }; +} + +seastar::future<> FSDriver::write( + off_t offset, + bufferptr ptr) +{ + auto mapping = map_offset(offset); + ceph_assert(mapping.offset + ptr.length() <= config.object_size); + ceph::os::Transaction t; + bufferlist bl; + bl.append(ptr); + t.write( + mapping.chandle->get_cid(), + mapping.object, + mapping.offset, + ptr.length(), + bl, + 0); + return fs->do_transaction( + mapping.chandle, + std::move(t)); +} + +seastar::future FSDriver::read( + off_t offset, + size_t size) +{ + auto mapping = map_offset(offset); + ceph_assert((mapping.offset + size) <= config.object_size); + return fs->read( + mapping.chandle, + mapping.object, + mapping.offset, + size, + 0 + ).handle_error( + crimson::ct_error::enoent::handle([size](auto &e) { + bufferlist bl; + bl.append_zero(size); + return seastar::make_ready_future(std::move(bl)); + }), + crimson::ct_error::assert_all{"Unrecoverable error in FSDriver::read"} + ).then([this, size](auto &&bl) { + if (bl.length() < size) { + bl.append_zero(size - bl.length()); + } + return seastar::make_ready_future(std::move(bl)); + }); +} + +seastar::future<> FSDriver::mkfs() +{ + init(); + assert(fs); + return fs->start( + ).then([this] { + uuid_d uuid; + uuid.generate_random(); + return fs->mkfs(uuid); + }).then([this] { + return fs->stop(); + }).then([this] { + init(); + return fs->start(); + }).then([this] { + return fs->mount(); + }).then([this] { + return seastar::do_for_each( + boost::counting_iterator(0), + boost::counting_iterator(config.num_collections), + [this](auto i) { + return fs->create_new_collection(get_coll(i) + ).then([this, i](auto coll) { + ceph::os::Transaction t; + t.create_collection(get_coll(i), 0); + return fs->do_transaction(coll, std::move(t)); + }); + }); + }).then([this] { + return fs->umount(); + }).then([this] { + return fs->stop(); + }).then([this] { + fs.reset(); + return seastar::now(); + }); +} + +seastar::future<> FSDriver::mount() +{ + ceph_assert(config.path); + return ( + config.mkfs ? mkfs() : seastar::now() + ).then([this] { + init(); + return fs->start(); + }).then([this] { + return fs->mount(); + }).then([this] { + return seastar::do_for_each( + boost::counting_iterator(0), + boost::counting_iterator(config.num_collections), + [this](auto i) { + return fs->open_collection(get_coll(i) + ).then([this, i](auto ref) { + collections[i] = ref; + return seastar::now(); + }); + }); + }).then([this] { + return fs->stat(); + }).then([this](auto s) { + size = s.total; + }); +}; + +seastar::future<> FSDriver::close() +{ + collections.clear(); + return fs->umount( + ).then([this] { + return fs->stop(); + }).then([this] { + fs.reset(); + return seastar::now(); + }); +} + +void FSDriver::init() +{ + fs = FuturizedStore::create( + config.get_fs_type(), + *config.path, + crimson::common::local_conf().get_config_values()); +} diff --git a/src/crimson/tools/store_nbd/fs_driver.h b/src/crimson/tools/store_nbd/fs_driver.h new file mode 100644 index 0000000000000..1ef0797a97f42 --- /dev/null +++ b/src/crimson/tools/store_nbd/fs_driver.h @@ -0,0 +1,49 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "block_driver.h" + +#include "crimson/os/futurized_collection.h" +#include "crimson/os/futurized_store.h" + +class FSDriver final : public BlockDriver { +public: + FSDriver(config_t config) : config(config) {} + ~FSDriver() final {} + + bufferptr get_buffer(size_t size) final { + return ceph::buffer::create_page_aligned(size); + } + + seastar::future<> write( + off_t offset, + bufferptr ptr) final; + + seastar::future read( + off_t offset, + size_t size) final; + + size_t get_size() const { + return size; + } + + seastar::future<> mount() final; + + seastar::future<> close() final; + +private: + size_t size = 0; + const config_t config; + std::unique_ptr fs; + std::map collections; + + struct offset_mapping_t { + crimson::os::CollectionRef chandle; + ghobject_t object; + off_t offset; + }; + offset_mapping_t map_offset(off_t offset); + + seastar::future<> mkfs(); + void init(); +}; -- 2.39.5