From 990e2e76f6162bf287a9a33f4e4273728180f27d Mon Sep 17 00:00:00 2001 From: chunmei Date: Fri, 3 Mar 2023 04:19:21 +0000 Subject: [PATCH] cirmson/os: seperate futurizedStore interface into futurizedStore and futurizedShardStore and modification the other parts in osd and objectstore Signed-off-by: chunmei (cherry picked from commit 7a673d006da9fec985b2653a2c16b9f4581398f9) --- src/crimson/os/alienstore/alien_store.h | 7 +- src/crimson/os/cyanstore/cyan_store.cc | 96 +-- src/crimson/os/cyanstore/cyan_store.h | 202 ++--- src/crimson/os/futurized_store.cc | 15 +- src/crimson/os/futurized_store.h | 429 ++++------ src/crimson/os/seastore/seastore.cc | 518 +++++++----- src/crimson/os/seastore/seastore.h | 735 ++++++++++-------- src/crimson/os/seastore/segment_manager.cc | 8 +- .../os/seastore/segment_manager/block.cc | 5 +- src/crimson/osd/ec_backend.h | 2 +- src/crimson/osd/main.cc | 2 +- src/crimson/osd/osd.cc | 27 +- src/crimson/osd/osd_meta.cc | 2 +- src/crimson/osd/osd_meta.h | 6 +- src/crimson/osd/pg.cc | 2 +- src/crimson/osd/pg.h | 2 +- src/crimson/osd/pg_backend.cc | 12 +- src/crimson/osd/pg_backend.h | 8 +- src/crimson/osd/pg_meta.cc | 8 +- src/crimson/osd/pg_meta.h | 9 +- src/crimson/osd/pg_shard_manager.cc | 4 +- src/crimson/osd/pg_shard_manager.h | 7 +- src/crimson/osd/recovery_backend.h | 2 +- .../osd/replicated_recovery_backend.cc | 8 +- src/crimson/osd/shard_services.cc | 2 +- src/crimson/osd/shard_services.h | 4 +- src/crimson/tools/store_nbd/fs_driver.cc | 32 +- src/crimson/tools/store_nbd/fs_driver.h | 1 + src/osd/PGLog.cc | 14 +- src/osd/PGLog.h | 6 +- src/osd/SnapMapper.cc | 16 +- src/osd/SnapMapper.h | 2 +- .../seastore/transaction_manager_test_state.h | 29 +- 33 files changed, 1113 insertions(+), 1109 deletions(-) diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h index 3e6eb1918a0..79c19b29896 100644 --- a/src/crimson/os/alienstore/alien_store.h +++ b/src/crimson/os/alienstore/alien_store.h @@ -20,7 +20,8 @@ class Transaction; namespace crimson::os { using coll_core_t = FuturizedStore::coll_core_t; -class AlienStore final : public FuturizedStore { +class AlienStore final : public FuturizedStore, + public FuturizedStore::Shard { public: AlienStore(const std::string& type, const std::string& path, @@ -99,6 +100,10 @@ public: uint64_t off, uint64_t len) final; + FuturizedStore::Shard& get_sharded_store() final { + return *this; + } + private: template auto do_with_op_gate(Args&&... args) const { diff --git a/src/crimson/os/cyanstore/cyan_store.cc b/src/crimson/os/cyanstore/cyan_store.cc index daeb1e56767..f2a6018e36a 100644 --- a/src/crimson/os/cyanstore/cyan_store.cc +++ b/src/crimson/os/cyanstore/cyan_store.cc @@ -53,16 +53,12 @@ private: }; }; -seastar::future<> CyanStore::start() -{ - return shard_stores.start(path); -} - seastar::future CyanStore::stat() const { + ceph_assert(seastar::this_shard_id() == primary_core); logger().debug("{}", __func__); return shard_stores.map_reduce0( - [](const CyanStore::ShardStores &local_store) { + [](const CyanStore::Shard &local_store) { return local_store.get_used_bytes(); }, (uint64_t)0, @@ -78,6 +74,7 @@ seastar::future CyanStore::stat() const CyanStore::mkfs_ertr::future<> CyanStore::mkfs(uuid_d new_osd_fsid) { + ceph_assert(seastar::this_shard_id() == primary_core); static const char read_meta_errmsg[]{"read_meta"}; static const char parse_fsid_errmsg[]{"failed to parse fsid"}; static const char match_ofsid_errmsg[]{"unmatched osd_fsid"}; @@ -113,7 +110,7 @@ CyanStore::mkfs_ertr::future<> CyanStore::mkfs(uuid_d new_osd_fsid) }); } -seastar::future<> CyanStore::ShardStores::mkfs() +seastar::future<> CyanStore::Shard::mkfs() { std::string fn = path + "/collections" + std::to_string(seastar::this_shard_id()); @@ -123,9 +120,11 @@ seastar::future<> CyanStore::ShardStores::mkfs() return crimson::write_file(std::move(bl), fn); } +using coll_core_t = FuturizedStore::coll_core_t; seastar::future> CyanStore::list_collections() { + ceph_assert(seastar::this_shard_id() == primary_core); return seastar::do_with(std::vector{}, [this](auto &collections) { return shard_stores.map([](auto &local_store) { return local_store.list_collections(); @@ -139,7 +138,7 @@ CyanStore::list_collections() }); } -CyanStore::mount_ertr::future<> CyanStore::ShardStores::mount() +CyanStore::mount_ertr::future<> CyanStore::Shard::mount() { static const char read_file_errmsg[]{"read_file"}; ceph::bufferlist bl; @@ -170,7 +169,7 @@ CyanStore::mount_ertr::future<> CyanStore::ShardStores::mount() return mount_ertr::now(); } -seastar::future<> CyanStore::ShardStores::umount() +seastar::future<> CyanStore::Shard::umount() { return seastar::do_with(std::set{}, [this](auto& collections) { return seastar::do_for_each(coll_map, [&collections, this](auto& coll) { @@ -193,7 +192,7 @@ seastar::future<> CyanStore::ShardStores::umount() } seastar::future, ghobject_t>> -CyanStore::ShardStores::list_objects( +CyanStore::Shard::list_objects( CollectionRef ch, const ghobject_t& start, const ghobject_t& end, @@ -220,7 +219,7 @@ CyanStore::ShardStores::list_objects( } seastar::future -CyanStore::ShardStores::create_new_collection(const coll_t& cid) +CyanStore::Shard::create_new_collection(const coll_t& cid) { auto c = new Collection{cid}; new_coll_map[cid] = c; @@ -228,13 +227,13 @@ CyanStore::ShardStores::create_new_collection(const coll_t& cid) } seastar::future -CyanStore::ShardStores::open_collection(const coll_t& cid) +CyanStore::Shard::open_collection(const coll_t& cid) { return seastar::make_ready_future(_get_collection(cid)); } seastar::future> -CyanStore::ShardStores::list_collections() +CyanStore::Shard::list_collections() { std::vector collections; for (auto& coll : coll_map) { @@ -243,8 +242,8 @@ CyanStore::ShardStores::list_collections() return seastar::make_ready_future>(std::move(collections)); } -CyanStore::read_errorator::future -CyanStore::ShardStores::read( +CyanStore::Shard::read_errorator::future +CyanStore::Shard::read( CollectionRef ch, const ghobject_t& oid, uint64_t offset, @@ -271,8 +270,8 @@ CyanStore::ShardStores::read( return read_errorator::make_ready_future(o->read(offset, l)); } -CyanStore::read_errorator::future -CyanStore::ShardStores::readv( +CyanStore::Shard::read_errorator::future +CyanStore::Shard::readv( CollectionRef ch, const ghobject_t& oid, interval_set& m, @@ -292,8 +291,8 @@ CyanStore::ShardStores::readv( }); } -CyanStore::get_attr_errorator::future -CyanStore::ShardStores::get_attr( +CyanStore::Shard::get_attr_errorator::future +CyanStore::Shard::get_attr( CollectionRef ch, const ghobject_t& oid, std::string_view name) const @@ -312,8 +311,8 @@ CyanStore::ShardStores::get_attr( } } -CyanStore::get_attrs_ertr::future -CyanStore::ShardStores::get_attrs( +CyanStore::Shard::get_attrs_ertr::future +CyanStore::Shard::get_attrs( CollectionRef ch, const ghobject_t& oid) { @@ -327,7 +326,7 @@ CyanStore::ShardStores::get_attrs( return get_attrs_ertr::make_ready_future(o->xattr); } -auto CyanStore::ShardStores::omap_get_values( +auto CyanStore::Shard::omap_get_values( CollectionRef ch, const ghobject_t& oid, const omap_keys_t& keys) @@ -348,11 +347,11 @@ auto CyanStore::ShardStores::omap_get_values( return seastar::make_ready_future(std::move(values)); } -auto CyanStore::ShardStores::omap_get_values( +auto CyanStore::Shard::omap_get_values( CollectionRef ch, const ghobject_t &oid, const std::optional &start) - -> read_errorator::future> + -> CyanStore::Shard::read_errorator::future> { auto c = static_cast(ch.get()); logger().debug("{} {} {}", __func__, c->get_cid(), oid); @@ -370,10 +369,10 @@ auto CyanStore::ShardStores::omap_get_values( std::make_tuple(true, std::move(values))); } -auto CyanStore::ShardStores::omap_get_header( +auto CyanStore::Shard::omap_get_header( CollectionRef ch, const ghobject_t& oid) - -> get_attr_errorator::future + -> CyanStore::Shard::get_attr_errorator::future { auto c = static_cast(ch.get()); auto o = c->get_object(oid); @@ -385,7 +384,7 @@ auto CyanStore::ShardStores::omap_get_header( o->omap_header); } -seastar::future<> CyanStore::ShardStores::do_transaction_no_callbacks( +seastar::future<> CyanStore::Shard::do_transaction_no_callbacks( CollectionRef ch, ceph::os::Transaction&& t) { @@ -577,7 +576,7 @@ seastar::future<> CyanStore::ShardStores::do_transaction_no_callbacks( return seastar::now(); } -int CyanStore::ShardStores::_remove(const coll_t& cid, const ghobject_t& oid) +int CyanStore::Shard::_remove(const coll_t& cid, const ghobject_t& oid) { logger().debug("{} cid={} oid={}", __func__, cid, oid); @@ -594,7 +593,7 @@ int CyanStore::ShardStores::_remove(const coll_t& cid, const ghobject_t& oid) return 0; } -int CyanStore::ShardStores::_touch(const coll_t& cid, const ghobject_t& oid) +int CyanStore::Shard::_touch(const coll_t& cid, const ghobject_t& oid) { logger().debug("{} cid={} oid={}", __func__, cid, oid); @@ -606,7 +605,7 @@ int CyanStore::ShardStores::_touch(const coll_t& cid, const ghobject_t& oid) return 0; } -int CyanStore::ShardStores::_write( +int CyanStore::Shard::_write( const coll_t& cid, const ghobject_t& oid, uint64_t offset, @@ -632,7 +631,7 @@ int CyanStore::ShardStores::_write( return 0; } -int CyanStore::ShardStores::_zero( +int CyanStore::Shard::_zero( const coll_t& cid, const ghobject_t& oid, uint64_t offset, @@ -646,7 +645,7 @@ int CyanStore::ShardStores::_zero( return _write(cid, oid, offset, len, bl, 0); } -int CyanStore::ShardStores::_omap_clear( +int CyanStore::Shard::_omap_clear( const coll_t& cid, const ghobject_t& oid) { @@ -665,7 +664,7 @@ int CyanStore::ShardStores::_omap_clear( return 0; } -int CyanStore::ShardStores::_omap_set_values( +int CyanStore::Shard::_omap_set_values( const coll_t& cid, const ghobject_t& oid, std::map &&aset) @@ -685,7 +684,7 @@ int CyanStore::ShardStores::_omap_set_values( return 0; } -int CyanStore::ShardStores::_omap_set_header( +int CyanStore::Shard::_omap_set_header( const coll_t& cid, const ghobject_t& oid, const ceph::bufferlist &header) @@ -703,7 +702,7 @@ int CyanStore::ShardStores::_omap_set_header( return 0; } -int CyanStore::ShardStores::_omap_rmkeys( +int CyanStore::Shard::_omap_rmkeys( const coll_t& cid, const ghobject_t& oid, const omap_keys_t& aset) @@ -723,7 +722,7 @@ int CyanStore::ShardStores::_omap_rmkeys( return 0; } -int CyanStore::ShardStores::_omap_rmkeyrange( +int CyanStore::Shard::_omap_rmkeyrange( const coll_t& cid, const ghobject_t& oid, const std::string &first, @@ -744,7 +743,7 @@ int CyanStore::ShardStores::_omap_rmkeyrange( return 0; } -int CyanStore::ShardStores::_truncate( +int CyanStore::Shard::_truncate( const coll_t& cid, const ghobject_t& oid, uint64_t size) @@ -766,7 +765,7 @@ int CyanStore::ShardStores::_truncate( return r; } -int CyanStore::ShardStores::_clone( +int CyanStore::Shard::_clone( const coll_t& cid, const ghobject_t& oid, const ghobject_t& noid) @@ -792,7 +791,7 @@ int CyanStore::ShardStores::_clone( return 0; } -int CyanStore::ShardStores::_setattrs( +int CyanStore::Shard::_setattrs( const coll_t& cid, const ghobject_t& oid, std::map&& aset) @@ -812,7 +811,7 @@ int CyanStore::ShardStores::_setattrs( return 0; } -int CyanStore::ShardStores::_rm_attr( +int CyanStore::Shard::_rm_attr( const coll_t& cid, const ghobject_t& oid, std::string_view name) @@ -834,7 +833,7 @@ int CyanStore::ShardStores::_rm_attr( return 0; } -int CyanStore::ShardStores::_rm_attrs( +int CyanStore::Shard::_rm_attrs( const coll_t& cid, const ghobject_t& oid) { @@ -851,7 +850,7 @@ int CyanStore::ShardStores::_rm_attrs( return 0; } -int CyanStore::ShardStores::_create_collection(const coll_t& cid, int bits) +int CyanStore::Shard::_create_collection(const coll_t& cid, int bits) { auto result = coll_map.try_emplace(cid); if (!result.second) @@ -865,7 +864,7 @@ int CyanStore::ShardStores::_create_collection(const coll_t& cid, int bits) } boost::intrusive_ptr -CyanStore::ShardStores::_get_collection(const coll_t& cid) +CyanStore::Shard::_get_collection(const coll_t& cid) { auto cp = coll_map.find(cid); if (cp == coll_map.end()) @@ -877,6 +876,7 @@ seastar::future<> CyanStore::write_meta( const std::string& key, const std::string& value) { + ceph_assert(seastar::this_shard_id() == primary_core); std::string v = value; v += "\n"; if (int r = safe_write_file(path.c_str(), key.c_str(), @@ -890,6 +890,7 @@ seastar::future<> CyanStore::write_meta( seastar::future> CyanStore::read_meta(const std::string& key) { + ceph_assert(seastar::this_shard_id() == primary_core); std::string fsid(4096, '\0'); int r = safe_read_file(path.c_str(), key.c_str(), fsid.data(), fsid.size()); if (r > 0) { @@ -906,17 +907,18 @@ CyanStore::read_meta(const std::string& key) uuid_d CyanStore::get_fsid() const { + ceph_assert(seastar::this_shard_id() == primary_core); return osd_fsid; } -unsigned CyanStore::get_max_attr_name_length() const +unsigned CyanStore::Shard::get_max_attr_name_length() const { // arbitrary limitation exactly like in the case of MemStore. return 256; } -CyanStore::read_errorator::future> -CyanStore::ShardStores::fiemap( +CyanStore::Shard::read_errorator::future> +CyanStore::Shard::fiemap( CollectionRef ch, const ghobject_t& oid, uint64_t off, @@ -933,7 +935,7 @@ CyanStore::ShardStores::fiemap( } seastar::future -CyanStore::ShardStores::stat( +CyanStore::Shard::stat( CollectionRef ch, const ghobject_t& oid) { diff --git a/src/crimson/os/cyanstore/cyan_store.h b/src/crimson/os/cyanstore/cyan_store.h index 715f8ae8caf..307f9ec32ed 100644 --- a/src/crimson/os/cyanstore/cyan_store.h +++ b/src/crimson/os/cyanstore/cyan_store.h @@ -25,85 +25,89 @@ class Transaction; } namespace crimson::os { -using coll_core_t = FuturizedStore::coll_core_t; class CyanStore final : public FuturizedStore { - class ShardStores { + class Shard : public FuturizedStore::Shard { public: - ShardStores(std::string path) + Shard(std::string path) :path(path){} - mount_ertr::future<> mount(); - - seastar::future<> umount(); - - seastar::future<> mkfs(); - - mkfs_ertr::future<> mkcoll(uuid_d new_osd_fsid); - seastar::future stat( CollectionRef c, - const ghobject_t& oid); + const ghobject_t& oid) final; read_errorator::future read( CollectionRef c, const ghobject_t& oid, uint64_t offset, size_t len, - uint32_t op_flags = 0); + uint32_t op_flags = 0) final; read_errorator::future readv( CollectionRef c, const ghobject_t& oid, interval_set& m, - uint32_t op_flags = 0); + uint32_t op_flags = 0) final; get_attr_errorator::future get_attr( CollectionRef c, const ghobject_t& oid, - std::string_view name) const; + std::string_view name) const final; get_attrs_ertr::future get_attrs( CollectionRef c, - const ghobject_t& oid); + const ghobject_t& oid) final; read_errorator::future omap_get_values( CollectionRef c, const ghobject_t& oid, - const omap_keys_t& keys); + const omap_keys_t& keys) final; read_errorator::future> omap_get_values( CollectionRef c, ///< [in] collection const ghobject_t &oid, ///< [in] oid const std::optional &start ///< [in] start, empty for begin - ); + ) final; + + get_attr_errorator::future omap_get_header( + CollectionRef c, + const ghobject_t& oid) final; seastar::future, ghobject_t>> list_objects( CollectionRef c, const ghobject_t& start, const ghobject_t& end, - uint64_t limit) const; + uint64_t limit) const final; - get_attr_errorator::future omap_get_header( - CollectionRef c, - const ghobject_t& oid); - - seastar::future create_new_collection(const coll_t& cid); + seastar::future create_new_collection(const coll_t& cid) final; - seastar::future open_collection(const coll_t& cid); - - seastar::future> list_collections(); + seastar::future open_collection(const coll_t& cid) final; seastar::future<> do_transaction_no_callbacks( CollectionRef ch, - ceph::os::Transaction&& txn); + ceph::os::Transaction&& txn) final; read_errorator::future> fiemap( CollectionRef c, const ghobject_t& oid, uint64_t off, - uint64_t len); + uint64_t len) final; + + unsigned get_max_attr_name_length() const final; + + public: + // only exposed to CyanStore + mount_ertr::future<> mount(); + + seastar::future<> umount(); + + seastar::future<> mkfs(); + + mkfs_ertr::future<> mkcoll(uuid_d new_osd_fsid); + + using coll_core_t = FuturizedStore::coll_core_t; + seastar::future> list_collections(); uint64_t get_used_bytes() const { return used_bytes; } @@ -137,11 +141,11 @@ class CyanStore final : public FuturizedStore { const std::string &last); int _truncate(const coll_t& cid, const ghobject_t& oid, uint64_t size); int _clone(const coll_t& cid, const ghobject_t& oid, - const ghobject_t& noid); + const ghobject_t& noid); int _setattrs(const coll_t& cid, const ghobject_t& oid, - std::map&& aset); + std::map&& aset); int _rm_attr(const coll_t& cid, const ghobject_t& oid, - std::string_view name); + std::string_view name); int _rm_attrs(const coll_t& cid, const ghobject_t& oid); int _create_collection(const coll_t& cid, int bits); boost::intrusive_ptr _get_collection(const coll_t& cid); @@ -153,29 +157,22 @@ class CyanStore final : public FuturizedStore { std::map> new_coll_map; }; -// public interfaces called by main OSD public: CyanStore(const std::string& path); ~CyanStore() final; + seastar::future<> start() final { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.start(path); + } + seastar::future<> stop() final { + ceph_assert(seastar::this_shard_id() == primary_core); return shard_stores.stop(); } - seastar::future<> start() final; - - mkfs_ertr::future<> mkfs(uuid_d new_osd_fsid) final; - - seastar::future stat() const final; - - seastar::future<> write_meta(const std::string& key, - const std::string& value) final; - - seastar::future> - read_meta(const std::string& key) final; - - uuid_d get_fsid() const final; mount_ertr::future<> mount() final { + ceph_assert(seastar::this_shard_id() == primary_core); return shard_stores.invoke_on_all( [](auto &local_store) { return local_store.mount().handle_error( @@ -189,123 +186,34 @@ public: } seastar::future<> umount() final { + ceph_assert(seastar::this_shard_id() == primary_core); return shard_stores.invoke_on_all( [](auto &local_store) { return local_store.umount(); }); } - seastar::future> list_collections() final; - -// public interfaces called by each shard osd -public: - unsigned get_max_attr_name_length() const final; - - seastar::future stat( - CollectionRef c, - const ghobject_t& oid) final { - return shard_stores.local().stat( - c, oid); - } - - read_errorator::future read( - CollectionRef c, - const ghobject_t& oid, - uint64_t offset, - size_t len, - uint32_t op_flags = 0) final { - return shard_stores.local().read( - c, oid, offset, len, op_flags); - } - - read_errorator::future readv( - CollectionRef c, - const ghobject_t& oid, - interval_set& m, - uint32_t op_flags = 0) final { - return shard_stores.local().readv( - c, oid, m, op_flags); - } - - get_attr_errorator::future get_attr( - CollectionRef c, - const ghobject_t& oid, - std::string_view name) const final { - return shard_stores.local().get_attr( - c, oid, name); - } - - get_attrs_ertr::future get_attrs( - CollectionRef c, - const ghobject_t& oid) final { - return shard_stores.local().get_attrs( - c, oid); - } - - read_errorator::future omap_get_values( - CollectionRef c, - const ghobject_t& oid, - const omap_keys_t& keys) final { - return shard_stores.local().omap_get_values( - c, oid, keys); - } - - /// Retrieves paged set of values > start (if present) - read_errorator::future> omap_get_values( - CollectionRef c, ///< [in] collection - const ghobject_t &oid, ///< [in] oid - const std::optional &start ///< [in] start, empty for begin - ) final { - return shard_stores.local().omap_get_values( - c, oid, start); - } + mkfs_ertr::future<> mkfs(uuid_d new_osd_fsid) final; - seastar::future, ghobject_t>> - list_objects( - CollectionRef c, - const ghobject_t& start, - const ghobject_t& end, - uint64_t limit) const final { - return shard_stores.local().list_objects( - c, start, end, limit); - } + seastar::future stat() const final; - get_attr_errorator::future omap_get_header( - CollectionRef c, - const ghobject_t& oid) final { - return shard_stores.local().omap_get_header( - c, oid); - } + uuid_d get_fsid() const final; - seastar::future - create_new_collection(const coll_t& cid) final { - return shard_stores.local().create_new_collection(cid); - } + seastar::future<> write_meta(const std::string& key, + const std::string& value) final; - seastar::future - open_collection(const coll_t& cid) final { - return shard_stores.local().open_collection(cid); + FuturizedStore::Shard& get_sharded_store() final{ + return shard_stores.local(); } - seastar::future<> do_transaction_no_callbacks( - CollectionRef ch, - ceph::os::Transaction&& txn) final { - return shard_stores.local().do_transaction_no_callbacks( - ch, std::move(txn)); - } + seastar::future> + read_meta(const std::string& key) final; - read_errorator::future> - fiemap(CollectionRef c, - const ghobject_t& oid, - uint64_t off, - uint64_t len) final { - return shard_stores.local().fiemap( - c, oid, off, len); - } + seastar::future> list_collections() final; private: + seastar::sharded shard_stores; const std::string path; uuid_d osd_fsid; - seastar::sharded shard_stores; }; } diff --git a/src/crimson/os/futurized_store.cc b/src/crimson/os/futurized_store.cc index b9542a50c60..bc47ec78fde 100644 --- a/src/crimson/os/futurized_store.cc +++ b/src/crimson/os/futurized_store.cc @@ -10,29 +10,22 @@ namespace crimson::os { -seastar::future> +std::unique_ptr FuturizedStore::create(const std::string& type, const std::string& data, const ConfigValues& values) { if (type == "cyanstore") { using crimson::os::CyanStore; - return seastar::make_ready_future>( - std::make_unique(data)); + return std::make_unique(data); } else if (type == "seastore") { return crimson::os::seastore::make_seastore( - data, values - ).then([] (auto seastore) { - return seastar::make_ready_future>( - std::make_unique>( - seastore.release())); - }); + data); } else { using crimson::os::AlienStore; #ifdef WITH_BLUESTORE // use AlienStore as a fallback. It adapts e.g. BlueStore. - return seastar::make_ready_future>( - std::make_unique(type, data, values)); + return std::make_unique(type, data, values); #else ceph_abort_msgf("unsupported objectstore type: %s", type.c_str()); return {}; diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h index 2fae5520275..1c65f5b2fda 100644 --- a/src/crimson/os/futurized_store.h +++ b/src/crimson/os/futurized_store.h @@ -25,322 +25,173 @@ class Transaction; namespace crimson::os { class FuturizedCollection; +constexpr core_id_t PRIMARY_CORE = 0; + class FuturizedStore { +public: + class Shard { + public: + Shard() = default; + virtual ~Shard() = default; + // no copying + explicit Shard(const Shard& o) = delete; + const Shard& operator=(const Shard& o) = delete; + + using CollectionRef = boost::intrusive_ptr; + using read_errorator = crimson::errorator; + virtual read_errorator::future read( + CollectionRef c, + const ghobject_t& oid, + uint64_t offset, + size_t len, + uint32_t op_flags = 0) = 0; + + virtual read_errorator::future readv( + CollectionRef c, + const ghobject_t& oid, + interval_set& m, + uint32_t op_flags = 0) = 0; + + using get_attr_errorator = crimson::errorator< + crimson::ct_error::enoent, + crimson::ct_error::enodata>; + virtual get_attr_errorator::future get_attr( + CollectionRef c, + const ghobject_t& oid, + std::string_view name) const = 0; + + using get_attrs_ertr = crimson::errorator< + crimson::ct_error::enoent>; + using attrs_t = std::map>; + virtual get_attrs_ertr::future get_attrs( + CollectionRef c, + const ghobject_t& oid) = 0; + + virtual seastar::future stat( + CollectionRef c, + const ghobject_t& oid) = 0; + + using omap_values_t = std::map>; + using omap_keys_t = std::set; + virtual read_errorator::future omap_get_values( + CollectionRef c, + const ghobject_t& oid, + const omap_keys_t& keys) = 0; + + virtual read_errorator::future> omap_get_values( + CollectionRef c, ///< [in] collection + const ghobject_t &oid, ///< [in] oid + const std::optional &start ///< [in] start, empty for begin + ) = 0; ///< @return values.empty() only if done + + virtual get_attr_errorator::future omap_get_header( + CollectionRef c, + const ghobject_t& oid) = 0; + + virtual seastar::future, ghobject_t>> list_objects( + CollectionRef c, + const ghobject_t& start, + const ghobject_t& end, + uint64_t limit) const = 0; + + virtual seastar::future create_new_collection(const coll_t& cid) = 0; + + virtual seastar::future open_collection(const coll_t& cid) = 0; + + protected: + virtual seastar::future<> do_transaction_no_callbacks( + CollectionRef ch, + ceph::os::Transaction&& txn) = 0; + + public: + seastar::future<> do_transaction( + CollectionRef ch, + ceph::os::Transaction&& txn) { + std::unique_ptr on_commit( + ceph::os::Transaction::collect_all_contexts(txn)); + return do_transaction_no_callbacks( + std::move(ch), std::move(txn) + ).then([on_commit=std::move(on_commit)]() mutable { + auto c = on_commit.release(); + if (c) c->complete(0); + return seastar::now(); + }); + } + + + /** + * flush + * + * Flushes outstanding transactions on ch, returned future resolves + * after any previously submitted transactions on ch have committed. + * + * @param ch [in] collection on which to flush + */ + virtual seastar::future<> flush(CollectionRef ch) { + return do_transaction(ch, ceph::os::Transaction{}); + } + + // error injection + virtual seastar::future<> inject_data_error(const ghobject_t& o) { + return seastar::now(); + } + + virtual seastar::future<> inject_mdata_error(const ghobject_t& o) { + return seastar::now(); + } + + virtual read_errorator::future> fiemap( + CollectionRef ch, + const ghobject_t& oid, + uint64_t off, + uint64_t len) = 0; + + virtual unsigned get_max_attr_name_length() const = 0; + }; public: - static seastar::future> create(const std::string& type, + static std::unique_ptr create(const std::string& type, const std::string& data, const ConfigValues& values); - FuturizedStore() = default; + FuturizedStore() + : primary_core(seastar::this_shard_id()) + {} + virtual ~FuturizedStore() = default; // no copying explicit FuturizedStore(const FuturizedStore& o) = delete; const FuturizedStore& operator=(const FuturizedStore& o) = delete; - virtual seastar::future<> start() { - return seastar::now(); - } + virtual seastar::future<> start() = 0; + virtual seastar::future<> stop() = 0; using mount_ertr = crimson::errorator; virtual mount_ertr::future<> mount() = 0; + virtual seastar::future<> umount() = 0; using mkfs_ertr = crimson::errorator; virtual mkfs_ertr::future<> mkfs(uuid_d new_osd_fsid) = 0; - virtual seastar::future stat() const = 0; - using CollectionRef = boost::intrusive_ptr; - using read_errorator = crimson::errorator; - virtual read_errorator::future read( - CollectionRef c, - const ghobject_t& oid, - uint64_t offset, - size_t len, - uint32_t op_flags = 0) = 0; - virtual read_errorator::future readv( - CollectionRef c, - const ghobject_t& oid, - interval_set& m, - uint32_t op_flags = 0) = 0; - - using get_attr_errorator = crimson::errorator< - crimson::ct_error::enoent, - crimson::ct_error::enodata>; - virtual get_attr_errorator::future get_attr( - CollectionRef c, - const ghobject_t& oid, - std::string_view name) const = 0; - - using get_attrs_ertr = crimson::errorator< - crimson::ct_error::enoent>; - using attrs_t = std::map>; - virtual get_attrs_ertr::future get_attrs( - CollectionRef c, - const ghobject_t& oid) = 0; - virtual seastar::future stat( - CollectionRef c, - const ghobject_t& oid) = 0; - - using omap_values_t = std::map>; - using omap_keys_t = std::set; - virtual read_errorator::future omap_get_values( - CollectionRef c, - const ghobject_t& oid, - const omap_keys_t& keys) = 0; - virtual seastar::future, ghobject_t>> list_objects( - CollectionRef c, - const ghobject_t& start, - const ghobject_t& end, - uint64_t limit) const = 0; - virtual read_errorator::future> omap_get_values( - CollectionRef c, ///< [in] collection - const ghobject_t &oid, ///< [in] oid - const std::optional &start ///< [in] start, empty for begin - ) = 0; ///< @return values.empty() only if done - - virtual get_attr_errorator::future omap_get_header( - CollectionRef c, - const ghobject_t& oid) = 0; - - virtual seastar::future create_new_collection(const coll_t& cid) = 0; - virtual seastar::future open_collection(const coll_t& cid) = 0; - using coll_core_t = std::pair; - virtual seastar::future> list_collections() = 0; - -protected: - virtual seastar::future<> do_transaction_no_callbacks( - CollectionRef ch, - ceph::os::Transaction&& txn) = 0; + virtual seastar::future stat() const = 0; -public: - seastar::future<> do_transaction( - CollectionRef ch, - ceph::os::Transaction&& txn) { - std::unique_ptr on_commit( - ceph::os::Transaction::collect_all_contexts(txn)); - return do_transaction_no_callbacks( - std::move(ch), std::move(txn) - ).then([on_commit=std::move(on_commit)]() mutable { - auto c = on_commit.release(); - if (c) c->complete(0); - return seastar::now(); - }); - } - - - /** - * flush - * - * Flushes outstanding transactions on ch, returned future resolves - * after any previously submitted transactions on ch have committed. - * - * @param ch [in] collection on which to flush - */ - virtual seastar::future<> flush(CollectionRef ch) { - return do_transaction(ch, ceph::os::Transaction{}); - } - - // error injection - virtual seastar::future<> inject_data_error(const ghobject_t& o) { - return seastar::now(); - } - virtual seastar::future<> inject_mdata_error(const ghobject_t& o) { - return seastar::now(); - } - - virtual read_errorator::future> fiemap( - CollectionRef ch, - const ghobject_t& oid, - uint64_t off, - uint64_t len) = 0; + virtual uuid_d get_fsid() const = 0; virtual seastar::future<> write_meta(const std::string& key, const std::string& value) = 0; + // called on the shard and get this FuturizedStore::shard; + virtual Shard& get_sharded_store() = 0; + virtual seastar::future> read_meta( const std::string& key) = 0; - virtual uuid_d get_fsid() const = 0; - virtual unsigned get_max_attr_name_length() const = 0; -}; - -/** - * ShardedStoreProxy - * - * Simple helper to proxy FuturizedStore operations to the core on which - * the store was initialized for implementations without support for multiple - * reactors. - */ -template -class ShardedStoreProxy : public FuturizedStore { - const core_id_t core; - std::unique_ptr impl; - uuid_d fsid; - unsigned max_attr = 0; - - template - decltype(auto) proxy(Method method, Args&&... args) const { - return proxy_method_on_core( - core, *impl, method, std::forward(args)...); - } - - template - decltype(auto) proxy(Method method, Args&&... args) { - return proxy_method_on_core( - core, *impl, method, std::forward(args)...); - } -public: - ShardedStoreProxy(T *t) - : core(seastar::this_shard_id()), - impl(t) {} - template - ShardedStoreProxy(Args&&... args) - : core(seastar::this_shard_id()), - impl(std::make_unique(std::forward(args)...)) {} - ~ShardedStoreProxy() = default; + using coll_core_t = std::pair; + virtual seastar::future> list_collections() = 0; - // no copying - explicit ShardedStoreProxy(const ShardedStoreProxy &o) = delete; - const ShardedStoreProxy &operator=(const ShardedStoreProxy &o) = delete; - - seastar::future<> start() final { return proxy(&T::start); } - seastar::future<> stop() final { return proxy(&T::stop); } - mount_ertr::future<> mount() final { - auto ret = seastar::smp::submit_to( - core, - [this] { - auto ret = impl->mount( - ).safe_then([this] { - fsid = impl->get_fsid(); - max_attr = impl->get_max_attr_name_length(); - return seastar::now(); - }); - return std::move(ret).to_base(); - }); - return mount_ertr::future<>(std::move(ret)); - } - seastar::future<> umount() final { return proxy(&T::umount); } - mkfs_ertr::future<> mkfs(uuid_d new_osd_fsid) final { - return proxy(&T::mkfs, new_osd_fsid); - } - seastar::future stat() const final { - return crimson::submit_to(core, [this] { return impl->stat(); }); - } - read_errorator::future read( - CollectionRef c, - const ghobject_t &oid, - uint64_t offset, - size_t len, - uint32_t op_flags = 0) final { - return proxy(&T::read, std::move(c), oid, offset, len, op_flags); - } - read_errorator::future readv( - CollectionRef c, - const ghobject_t &oid, - interval_set &m, - uint32_t op_flags = 0) final { - return crimson::submit_to(core, [this, c, oid, m, op_flags]() mutable { - return impl->readv(c, oid, m, op_flags); - }); - } - get_attr_errorator::future get_attr( - CollectionRef c, - const ghobject_t &oid, - std::string_view name) const final { - return proxy(&T::get_attr, std::move(c), oid, std::string(name)); - } - get_attrs_ertr::future get_attrs( - CollectionRef c, - const ghobject_t &oid) final { - return proxy(&T::get_attrs, std::move(c), oid); - } - seastar::future stat( - CollectionRef c, - const ghobject_t &oid) final { - return crimson::submit_to( - core, - [this, c, oid] { - return impl->stat(c, oid); - }); - } - read_errorator::future omap_get_values( - CollectionRef c, - const ghobject_t &oid, - const omap_keys_t &keys) final { - return crimson::submit_to(core, [this, c, oid, keys] { - return impl->omap_get_values(c, oid, keys); - }); - } - seastar::future, ghobject_t>> - list_objects( - CollectionRef c, - const ghobject_t &start, - const ghobject_t &end, - uint64_t limit) const final { - return proxy(&T::list_objects, std::move(c), start, end, limit); - } - read_errorator::future> omap_get_values( - CollectionRef c, - const ghobject_t &oid, - const std::optional &start) final { - return crimson::submit_to(core, [this, c, oid, start] { - return impl->omap_get_values(c, oid, start); - }); - } - get_attr_errorator::future omap_get_header( - CollectionRef c, - const ghobject_t &oid) final { - return proxy(&T::omap_get_header, std::move(c), oid); - } - seastar::future create_new_collection(const coll_t &cid) final { - return proxy(&T::create_new_collection, cid); - } - seastar::future open_collection(const coll_t &cid) final { - return proxy(&T::open_collection, cid); - } - seastar::future> list_collections() final { - return proxy(&T::list_collections); - } - seastar::future<> do_transaction_no_callbacks( - CollectionRef ch, - ceph::os::Transaction &&txn) final { - return proxy(&T::do_transaction_no_callbacks, std::move(ch), std::move(txn)); - } - seastar::future<> flush(CollectionRef ch) final { - return proxy(&T::flush, std::move(ch)); - } - seastar::future<> inject_data_error(const ghobject_t &o) final { - return proxy(&T::inject_data_error, o); - } - seastar::future<> inject_mdata_error(const ghobject_t &o) final { - return proxy(&T::inject_mdata_error, o); - } - - read_errorator::future> fiemap( - CollectionRef ch, - const ghobject_t &oid, - uint64_t off, - uint64_t len) final { - return proxy(&T::fiemap, std::move(ch), oid, off, len); - } - - seastar::future<> write_meta( - const std::string &key, - const std::string &value) final { - return proxy(&T::write_meta, key, value); - } - seastar::future> read_meta( - const std::string &key) final { - return proxy(&T::read_meta, key); - } - uuid_d get_fsid() const final { - return fsid; - } - unsigned get_max_attr_name_length() const final { - return max_attr; - } +protected: + const core_id_t primary_core; }; - } diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 218c0075fde..95cef4d1d0e 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -34,9 +34,9 @@ using std::string; using crimson::common::local_conf; -template <> struct fmt::formatter +template <> struct fmt::formatter : fmt::formatter { - using op_type_t = crimson::os::seastore::SeaStore::op_type_t; + using op_type_t = crimson::os::seastore::op_type_t; // parse is inherited from formatter. template auto format(op_type_t op, FormatContext& ctx) { @@ -113,29 +113,35 @@ public: using crimson::common::get_conf; -SeaStore::SeaStore( - const std::string& root, - MDStoreRef mdstore, - DeviceRef dev, +SeaStore::Shard::Shard( + std::string root, + Device* dev, bool is_test) - : root(root), - mdstore(std::move(mdstore)), - device(std::move(dev)), - max_object_size( - get_conf("seastore_default_max_object_size")), - is_test(is_test), - throttler( + :root(root), + max_object_size( + get_conf("seastore_default_max_object_size")), + is_test(is_test), + throttler( get_conf("seastore_max_concurrent_transactions")) { + device.reset(dev); register_metrics(); } +SeaStore::SeaStore( + const std::string& root, + MDStoreRef mdstore) + : root(root), + mdstore(std::move(mdstore)) +{ +} + SeaStore::~SeaStore() = default; -void SeaStore::register_metrics() +void SeaStore::Shard::register_metrics() { namespace sm = seastar::metrics; - using op_type_t = SeaStore::op_type_t; + using op_type_t = crimson::os::seastore::op_type_t; std::pair labels_by_op_type[] = { {op_type_t::TRANSACTION, sm::label_instance("latency", "TRANSACTION")}, {op_type_t::READ, sm::label_instance("latency", "READ")}, @@ -186,23 +192,71 @@ void SeaStore::register_metrics() ); } +seastar::future<> SeaStore::start() +{ + ceph_assert(seastar::this_shard_id() == primary_core); +#ifndef NDEBUG + bool is_test = true; +#else + bool is_test = false; +#endif + return shard_stores.start(root, nullptr, is_test) + .then([this] { + return shard_stores.invoke_on_all([](auto& local_store) { + return local_store.make_shard_stores(); + }); + }); +} + +seastar::future<> SeaStore::test_start(DeviceRef device) +{ + if (device) { + ceph_assert(root == ""); + return shard_stores.start_single(root, device.release(), true); + } else { + ceph_assert(0 == "impossible no device"); + } +} + + seastar::future<> SeaStore::stop() { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.stop(); +} + +seastar::future<> SeaStore::Shard::make_shard_stores() +{ + if (root != "") { + using crimson::common::get_conf; + std::string type = get_conf("seastore_main_device_type"); + device_type_t d_type = string_to_device_type(type); + assert(d_type == device_type_t::SSD || + d_type == device_type_t::RANDOM_BLOCK_SSD); + + return Device::make_device( + root, d_type + ).then([this](DeviceRef device_obj) { + device = std::move(device_obj); + }); + } return seastar::now(); } SeaStore::mount_ertr::future<> SeaStore::test_mount() { - init_managers(); - return transaction_manager->mount( - ).handle_error( - crimson::ct_error::assert_all{ - "Invalid error in SeaStore::test_mount" - } - ); + + ceph_assert(seastar::this_shard_id() == primary_core); + shard_stores.local().init_managers(); + return shard_stores.local().get_transaction_manager()->mount( + ).handle_error( + crimson::ct_error::assert_all{ + "Invalid error in SeaStore::test_mount" + } + ); } -SeaStore::mount_ertr::future<> SeaStore::mount() +SeaStore::mount_ertr::future<> SeaStore::Shard::mount() { return device->mount( ).safe_then([this] { @@ -211,9 +265,9 @@ SeaStore::mount_ertr::future<> SeaStore::mount() device_id_t id = device_entry.first; magic_t magic = device_entry.second.magic; device_type_t dtype = device_entry.second.dtype; - std::ostringstream oss; - oss << root << "/block." << dtype << "." << std::to_string(id); - return Device::make_device(oss.str(), dtype + std::string path = + fmt::format("{}/block.{}.{}", root, dtype, std::to_string(id)); + return Device::make_device(path, dtype ).then([this, magic](DeviceRef sec_dev) { return sec_dev->mount( ).safe_then([this, sec_dev=std::move(sec_dev), magic]() mutable { @@ -228,12 +282,12 @@ SeaStore::mount_ertr::future<> SeaStore::mount() return transaction_manager->mount(); }).handle_error( crimson::ct_error::assert_all{ - "Invalid error in SeaStore::mount" + "Invalid error in Shard::mount" } ); } -seastar::future<> SeaStore::umount() +seastar::future<> SeaStore::Shard::umount() { return [this] { if (transaction_manager) { @@ -264,6 +318,7 @@ seastar::future<> SeaStore::umount() seastar::future<> SeaStore::write_fsid(uuid_d new_osd_fsid) { + ceph_assert(seastar::this_shard_id() == primary_core); LOG_PREFIX(SeaStore::write_fsid); return read_meta("fsid").then([this, FNAME, new_osd_fsid] (auto tuple) { auto [ret, fsid] = tuple; @@ -280,7 +335,8 @@ seastar::future<> SeaStore::write_fsid(uuid_d new_osd_fsid) }); } -seastar::future<> SeaStore::_mkfs(uuid_d new_osd_fsid) +seastar::future<> +SeaStore::Shard::mkfs_managers() { init_managers(); return transaction_manager->mkfs( @@ -304,33 +360,88 @@ seastar::future<> SeaStore::_mkfs(uuid_d new_osd_fsid) }); }); }); - }).safe_then([this, new_osd_fsid] { - return write_fsid(new_osd_fsid); - }).safe_then([this] { - return read_meta("type").then([this] (auto tuple) { - auto [ret, type] = tuple; - if (ret == 0 && type == "seastore") { - return seastar::now(); - } else if (ret == 0 && type != "seastore") { - LOG_PREFIX(SeaStore::mkfs); - ERROR("expected seastore, but type is {}", type); - throw std::runtime_error("store type error"); - } else { - return write_meta("type", "seastore"); - } + }).handle_error( + crimson::ct_error::assert_all{ + "Invalid error in Shard::mkfs_managers" + } + ); +} + +seastar::future<> +SeaStore::Shard::mkfs( + secondary_device_set_t &sds, + uuid_d new_osd_fsid) +{ + device_type_t d_type = device->get_device_type(); + device_id_t id = (d_type == device_type_t::RANDOM_BLOCK_SSD) ? + static_cast(DEVICE_ID_RANDOM_BLOCK_MIN) : 0; + + return device->mkfs( + device_config_t{ + true, + device_spec_t{ + (magic_t)std::rand(), + d_type, + id}, + seastore_meta_t{new_osd_fsid}, + sds} + ).safe_then([this] { + return crimson::do_for_each(secondaries, [](auto& sec_dev) { + return sec_dev->mount(); }); }).safe_then([this] { - return write_meta("mkfs_done", "yes"); + return device->mount(); + }).safe_then([this] { + return mkfs_managers(); }).handle_error( crimson::ct_error::assert_all{ - "Invalid error in SeaStore::_mkfs" + "Invalid error in SeaStore::Shard::mkfs" } ); } -SeaStore::mkfs_ertr::future<> SeaStore::test_mkfs(uuid_d new_osd_fsid) +seastar::future<> SeaStore::Shard::sec_mkfs( + const std::string path, + device_type_t dtype, + device_id_t id, + secondary_device_set_t &sds, + uuid_d new_osd_fsid) +{ + return Device::make_device(path, dtype + ).then([this, &sds, id, dtype, new_osd_fsid](DeviceRef sec_dev) { + magic_t magic = (magic_t)std::rand(); + sds.emplace( + (device_id_t)id, + device_spec_t{magic, dtype, (device_id_t)id}); + return sec_dev->mkfs( + device_config_t{ + false, + device_spec_t{ + magic, + dtype, + (device_id_t)id}, + seastore_meta_t{new_osd_fsid}, + secondary_device_set_t()} + ).safe_then([this, sec_dev=std::move(sec_dev), id]() mutable { + LOG_PREFIX(SeaStore::Shard::sec_mkfs); + DEBUG("mkfs: finished for device {}", id); + secondaries.emplace_back(std::move(sec_dev)); + }).handle_error(crimson::ct_error::assert_all{"not possible"}); + }); +} + +seastar::future<> SeaStore::_mkfs(uuid_d new_osd_fsid) { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.local().mkfs_managers( + ).then([this, new_osd_fsid] { + return prepare_meta(new_osd_fsid); + }); +} +SeaStore::mkfs_ertr::future<> SeaStore::test_mkfs(uuid_d new_osd_fsid) +{ + ceph_assert(seastar::this_shard_id() == primary_core); return read_meta("mkfs_done").then([this, new_osd_fsid] (auto tuple) { auto [done, value] = tuple; if (done == 0) { @@ -340,16 +451,39 @@ SeaStore::mkfs_ertr::future<> SeaStore::test_mkfs(uuid_d new_osd_fsid) }); } +seastar::future<> SeaStore::prepare_meta(uuid_d new_osd_fsid) +{ + ceph_assert(seastar::this_shard_id() == primary_core); + return write_fsid(new_osd_fsid).then([this] { + return read_meta("type").then([this] (auto tuple) { + auto [ret, type] = tuple; + if (ret == 0 && type == "seastore") { + return seastar::now(); + } else if (ret == 0 && type != "seastore") { + LOG_PREFIX(SeaStore::prepare_meta); + ERROR("expected seastore, but type is {}", type); + throw std::runtime_error("store type error"); + } else { + return write_meta("type", "seastore"); + } + }); + }).then([this] { + return write_meta("mkfs_done", "yes"); + }); +} + SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) { + ceph_assert(seastar::this_shard_id() == primary_core); return read_meta("mkfs_done").then([this, new_osd_fsid] (auto tuple) { auto [done, value] = tuple; if (done == 0) { return seastar::now(); } else { return seastar::do_with( - secondary_device_set_t(), + std::vector(), [this, new_osd_fsid](auto& sds) { + sds.resize(seastar::smp::count); auto fut = seastar::now(); LOG_PREFIX(SeaStore::mkfs); DEBUG("root: {}", root); @@ -375,27 +509,16 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) return seastar::now(); } auto id = std::stoi(entry_name.substr(dtype_end + 1)); - std::ostringstream oss; - oss << root << "/" << entry_name; - return Device::make_device(oss.str(), dtype - ).then([this, &sds, id, dtype, new_osd_fsid](DeviceRef sec_dev) { - magic_t magic = (magic_t)std::rand(); - sds.emplace( - (device_id_t)id, - device_spec_t{magic, dtype, (device_id_t)id}); - return sec_dev->mkfs(device_config_t{ - false, - device_spec_t{ - magic, - dtype, - (device_id_t)id}, - seastore_meta_t{new_osd_fsid}, - secondary_device_set_t()} - ).safe_then([this, sec_dev=std::move(sec_dev), id]() mutable { - LOG_PREFIX(SeaStore::mkfs); - DEBUG("mkfs: finished for device {}", id); - secondaries.emplace_back(std::move(sec_dev)); - }).handle_error(crimson::ct_error::assert_all{"not possible"}); + std::string path = fmt::format("{}/{}", root, entry_name); + return shard_stores.invoke_on_all( + [&sds, id, path, dtype, new_osd_fsid] + (auto &local_store) { + return local_store.sec_mkfs( + path, + dtype, + id, + sds[seastar::this_shard_id()], + new_osd_fsid); }); } return seastar::now(); @@ -404,55 +527,64 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) }); } return fut.then([this, &sds, new_osd_fsid] { - device_id_t id = 0; - device_type_t d_type = device->get_device_type(); - assert(d_type == device_type_t::SSD || - d_type == device_type_t::RANDOM_BLOCK_SSD); - if (d_type == device_type_t::RANDOM_BLOCK_SSD) { - id = static_cast(DEVICE_ID_RANDOM_BLOCK_MIN); - } - - return device->mkfs( - device_config_t{ - true, - device_spec_t{ - (magic_t)std::rand(), - d_type, - id}, - seastore_meta_t{new_osd_fsid}, - sds} - ); - }).safe_then([this] { - return crimson::do_for_each(secondaries, [](auto& sec_dev) { - return sec_dev->mount(); + return shard_stores.invoke_on_all( + [&sds, new_osd_fsid](auto &local_store) { + return local_store.mkfs( + sds[seastar::this_shard_id()], new_osd_fsid); }); }); - }).safe_then([this] { - return device->mount(); - }).safe_then([this, new_osd_fsid] { - return _mkfs(new_osd_fsid); - }).safe_then([this] { + }).then([this, new_osd_fsid] { + return prepare_meta(new_osd_fsid); + }).then([this] { return umount(); - }).handle_error( - crimson::ct_error::assert_all{ - "Invalid error in SeaStore::mkfs" - } - ); + }); + } + }); +} + +using coll_core_t = FuturizedStore::coll_core_t; +seastar::future> +SeaStore::list_collections() +{ + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.map([](auto &local_store) { + return local_store.list_collections(); + }).then([](std::vector> results) { + std::vector collections; + for (auto& colls : results) { + collections.insert(collections.end(), colls.begin(), colls.end()); } + return seastar::make_ready_future>( + std::move(collections)); }); } +store_statfs_t SeaStore::Shard::stat() const +{ + return transaction_manager->store_stat(); +} + seastar::future SeaStore::stat() const { + ceph_assert(seastar::this_shard_id() == primary_core); LOG_PREFIX(SeaStore::stat); DEBUG(""); - return seastar::make_ready_future( - transaction_manager->store_stat() - ); + return shard_stores.map_reduce0( + [](const SeaStore::Shard &local_store) { + return local_store.stat(); + }, + store_statfs_t(), + [](auto &&ss, auto &&ret) { + ss.add(ret); + return std::move(ss); + } + ).then([](store_statfs_t ss) { + return seastar::make_ready_future(std::move(ss)); + }); } TransactionManager::read_extent_iertr::future> -SeaStore::get_coll_bits(CollectionRef ch, Transaction &t) const +SeaStore::Shard::get_coll_bits(CollectionRef ch, Transaction &t) const { return transaction_manager->read_collection_root(t) .si_then([this, ch, &t](auto coll_root) { @@ -545,7 +677,7 @@ get_ranges(CollectionRef ch, } seastar::future, ghobject_t>> -SeaStore::list_objects(CollectionRef ch, +SeaStore::Shard::list_objects(CollectionRef ch, const ghobject_t& start, const ghobject_t& end, uint64_t limit) const @@ -573,7 +705,7 @@ SeaStore::list_objects(CollectionRef ch, std::vector(), ghobject_t::get_max())); } else { - auto filter = get_objs_range(ch, *bits); + auto filter = SeaStore::get_objs_range(ch, *bits); using list_iertr = OnodeManager::list_onodes_iertr; using repeat_ret = list_iertr::future; return trans_intr::repeat( @@ -627,20 +759,23 @@ SeaStore::list_objects(CollectionRef ch, }); } -seastar::future SeaStore::create_new_collection(const coll_t& cid) +seastar::future +SeaStore::Shard::create_new_collection(const coll_t& cid) { LOG_PREFIX(SeaStore::create_new_collection); DEBUG("{}", cid); return seastar::make_ready_future(_get_collection(cid)); } -seastar::future SeaStore::open_collection(const coll_t& cid) +seastar::future +SeaStore::Shard::open_collection(const coll_t& cid) { LOG_PREFIX(SeaStore::open_collection); DEBUG("{}", cid); return list_collections().then([cid, this] (auto colls_cores) { - if (auto found = std::find(colls_cores.begin(), colls_cores.end(), - std::make_pair(cid, NULL_CORE)); + if (auto found = std::find(colls_cores.begin(), + colls_cores.end(), + std::make_pair(cid, seastar::this_shard_id())); found != colls_cores.end()) { return seastar::make_ready_future(_get_collection(cid)); } else { @@ -649,7 +784,8 @@ seastar::future SeaStore::open_collection(const coll_t& cid) }); } -seastar::future> SeaStore::list_collections() +seastar::future> +SeaStore::Shard::list_collections() { return seastar::do_with( std::vector(), @@ -667,7 +803,9 @@ seastar::future> SeaStore::list_collections() ret.resize(colls.size()); std::transform( colls.begin(), colls.end(), ret.begin(), - [](auto p) { return std::make_pair(p.first, NULL_CORE); }); + [](auto p) { + return std::make_pair(p.first, seastar::this_shard_id()); + }); }); }); }).safe_then([&ret] { @@ -681,7 +819,8 @@ seastar::future> SeaStore::list_collections() ); } -SeaStore::read_errorator::future SeaStore::read( +SeaStore::Shard::read_errorator::future +SeaStore::Shard::read( CollectionRef ch, const ghobject_t& oid, uint64_t offset, @@ -718,7 +857,8 @@ SeaStore::read_errorator::future SeaStore::read( }); } -SeaStore::read_errorator::future SeaStore::readv( +SeaStore::Shard::read_errorator::future +SeaStore::Shard::readv( CollectionRef ch, const ghobject_t& _oid, interval_set& m, @@ -746,7 +886,8 @@ SeaStore::read_errorator::future SeaStore::readv( using crimson::os::seastore::omap_manager::BtreeOMapManager; -SeaStore::get_attr_errorator::future SeaStore::get_attr( +SeaStore::Shard::get_attr_errorator::future +SeaStore::Shard::get_attr( CollectionRef ch, const ghobject_t& oid, std::string_view name) const @@ -784,7 +925,8 @@ SeaStore::get_attr_errorator::future SeaStore::get_attr( }), crimson::ct_error::pass_further_all{}); } -SeaStore::get_attrs_ertr::future SeaStore::get_attrs( +SeaStore::Shard::get_attrs_ertr::future +SeaStore::Shard::get_attrs( CollectionRef ch, const ghobject_t& oid) { @@ -822,7 +964,7 @@ SeaStore::get_attrs_ertr::future SeaStore::get_attrs( }), crimson::ct_error::pass_further_all{}); } -seastar::future SeaStore::stat( +seastar::future SeaStore::Shard::stat( CollectionRef c, const ghobject_t& oid) { @@ -850,16 +992,16 @@ seastar::future SeaStore::stat( ); } -SeaStore::get_attr_errorator::future -SeaStore::omap_get_header( +SeaStore::Shard::get_attr_errorator::future +SeaStore::Shard::omap_get_header( CollectionRef ch, const ghobject_t& oid) { return get_attr(ch, oid, OMAP_HEADER_XATTR_KEY); } -SeaStore::read_errorator::future -SeaStore::omap_get_values( +SeaStore::Shard::read_errorator::future +SeaStore::Shard::omap_get_values( CollectionRef ch, const ghobject_t &oid, const omap_keys_t &keys) @@ -881,7 +1023,8 @@ SeaStore::omap_get_values( }); } -SeaStore::_omap_get_value_ret SeaStore::_omap_get_value( +SeaStore::Shard::_omap_get_value_ret +SeaStore::Shard::_omap_get_value( Transaction &t, omap_root_t &&root, std::string_view key) const @@ -905,7 +1048,8 @@ SeaStore::_omap_get_value_ret SeaStore::_omap_get_value( ); } -SeaStore::_omap_get_values_ret SeaStore::_omap_get_values( +SeaStore::Shard::_omap_get_values_ret +SeaStore::Shard::_omap_get_values( Transaction &t, omap_root_t &&omap_root, const omap_keys_t &keys) const @@ -944,7 +1088,8 @@ SeaStore::_omap_get_values_ret SeaStore::_omap_get_values( ); } -SeaStore::omap_list_ret SeaStore::omap_list( +SeaStore::Shard::omap_list_ret +SeaStore::Shard::omap_list( Onode &onode, const omap_root_le_t& omap_root, Transaction& t, @@ -968,7 +1113,8 @@ SeaStore::omap_list_ret SeaStore::omap_list( }); } -SeaStore::omap_get_values_ret_t SeaStore::omap_get_values( +SeaStore::Shard::omap_get_values_ret_t +SeaStore::Shard::omap_get_values( CollectionRef ch, const ghobject_t &oid, const std::optional &start) @@ -976,7 +1122,7 @@ SeaStore::omap_get_values_ret_t SeaStore::omap_get_values( auto c = static_cast(ch.get()); LOG_PREFIX(SeaStore::omap_get_values); DEBUG("{} {}", c->get_cid(), oid); - using ret_bare_t = std::tuple; + using ret_bare_t = std::tuple; return repeat_with_onode( c, oid, @@ -993,7 +1139,7 @@ SeaStore::omap_get_values_ret_t SeaStore::omap_get_values( }); } -SeaStore::_fiemap_ret SeaStore::_fiemap( +SeaStore::Shard::_fiemap_ret SeaStore::Shard::_fiemap( Transaction &t, Onode &onode, uint64_t off, @@ -1013,7 +1159,8 @@ SeaStore::_fiemap_ret SeaStore::_fiemap( }); } -SeaStore::read_errorator::future> SeaStore::fiemap( +SeaStore::Shard::read_errorator::future> +SeaStore::Shard::fiemap( CollectionRef ch, const ghobject_t& oid, uint64_t off, @@ -1040,7 +1187,7 @@ SeaStore::read_errorator::future> SeaStore::fiemap( }); } -void SeaStore::on_error(ceph::os::Transaction &t) { +void SeaStore::Shard::on_error(ceph::os::Transaction &t) { LOG_PREFIX(SeaStore::on_error); ERROR(" transaction dump:\n"); JSONFormatter f(true); @@ -1053,7 +1200,7 @@ void SeaStore::on_error(ceph::os::Transaction &t) { abort(); } -seastar::future<> SeaStore::do_transaction_no_callbacks( +seastar::future<> SeaStore::Shard::do_transaction_no_callbacks( CollectionRef _ch, ceph::os::Transaction&& _t) { @@ -1095,7 +1242,7 @@ seastar::future<> SeaStore::do_transaction_no_callbacks( } -seastar::future<> SeaStore::flush(CollectionRef ch) +seastar::future<> SeaStore::Shard::flush(CollectionRef ch) { return seastar::do_with( get_dummy_ordering_handle(), @@ -1108,7 +1255,8 @@ seastar::future<> SeaStore::flush(CollectionRef ch) }); } -SeaStore::tm_ret SeaStore::_do_transaction_step( +SeaStore::Shard::tm_ret +SeaStore::Shard::_do_transaction_step( internal_context_t &ctx, CollectionRef &col, std::vector &onodes, @@ -1292,7 +1440,8 @@ SeaStore::tm_ret SeaStore::_do_transaction_step( ); } -SeaStore::tm_ret SeaStore::_remove( +SeaStore::Shard::tm_ret +SeaStore::Shard::_remove( internal_context_t &ctx, OnodeRef &onode) { @@ -1334,7 +1483,8 @@ SeaStore::tm_ret SeaStore::_remove( ); } -SeaStore::tm_ret SeaStore::_touch( +SeaStore::Shard::tm_ret +SeaStore::Shard::_touch( internal_context_t &ctx, OnodeRef &onode) { @@ -1343,7 +1493,8 @@ SeaStore::tm_ret SeaStore::_touch( return tm_iertr::now(); } -SeaStore::tm_ret SeaStore::_write( +SeaStore::Shard::tm_ret +SeaStore::Shard::_write( internal_context_t &ctx, OnodeRef &onode, uint64_t offset, size_t len, @@ -1373,7 +1524,8 @@ SeaStore::tm_ret SeaStore::_write( }); } -SeaStore::tm_ret SeaStore::_zero( +SeaStore::Shard::tm_ret +SeaStore::Shard::_zero( internal_context_t &ctx, OnodeRef &onode, objaddr_t offset, @@ -1400,8 +1552,8 @@ SeaStore::tm_ret SeaStore::_zero( }); } -SeaStore::omap_set_kvs_ret -SeaStore::_omap_set_kvs( +SeaStore::Shard::omap_set_kvs_ret +SeaStore::Shard::_omap_set_kvs( OnodeRef &onode, const omap_root_le_t& omap_root, Transaction& t, @@ -1434,7 +1586,8 @@ SeaStore::_omap_set_kvs( ); } -SeaStore::tm_ret SeaStore::_omap_set_values( +SeaStore::Shard::tm_ret +SeaStore::Shard::_omap_set_values( internal_context_t &ctx, OnodeRef &onode, std::map &&aset) @@ -1449,7 +1602,8 @@ SeaStore::tm_ret SeaStore::_omap_set_values( std::move(aset)); } -SeaStore::tm_ret SeaStore::_omap_set_header( +SeaStore::Shard::tm_ret +SeaStore::Shard::_omap_set_header( internal_context_t &ctx, OnodeRef &onode, ceph::bufferlist &&header) @@ -1461,7 +1615,8 @@ SeaStore::tm_ret SeaStore::_omap_set_header( return _setattrs(ctx, onode,std::move(to_set)); } -SeaStore::tm_ret SeaStore::_omap_clear( +SeaStore::Shard::tm_ret +SeaStore::Shard::_omap_clear( internal_context_t &ctx, OnodeRef &onode) { @@ -1495,7 +1650,8 @@ SeaStore::tm_ret SeaStore::_omap_clear( }); } -SeaStore::tm_ret SeaStore::_omap_rmkeys( +SeaStore::Shard::tm_ret +SeaStore::Shard::_omap_rmkeys( internal_context_t &ctx, OnodeRef &onode, omap_keys_t &&keys) @@ -1536,7 +1692,8 @@ SeaStore::tm_ret SeaStore::_omap_rmkeys( } } -SeaStore::tm_ret SeaStore::_omap_rmkeyrange( +SeaStore::Shard::tm_ret +SeaStore::Shard::_omap_rmkeyrange( internal_context_t &ctx, OnodeRef &onode, std::string first, @@ -1583,7 +1740,8 @@ SeaStore::tm_ret SeaStore::_omap_rmkeyrange( } } -SeaStore::tm_ret SeaStore::_truncate( +SeaStore::Shard::tm_ret +SeaStore::Shard::_truncate( internal_context_t &ctx, OnodeRef &onode, uint64_t size) @@ -1604,7 +1762,8 @@ SeaStore::tm_ret SeaStore::_truncate( }); } -SeaStore::tm_ret SeaStore::_setattrs( +SeaStore::Shard::tm_ret +SeaStore::Shard::_setattrs( internal_context_t &ctx, OnodeRef &onode, std::map&& aset) @@ -1671,7 +1830,8 @@ SeaStore::tm_ret SeaStore::_setattrs( }); } -SeaStore::tm_ret SeaStore::_rmattr( +SeaStore::Shard::tm_ret +SeaStore::Shard::_rmattr( internal_context_t &ctx, OnodeRef &onode, std::string name) @@ -1695,7 +1855,8 @@ SeaStore::tm_ret SeaStore::_rmattr( } } -SeaStore::tm_ret SeaStore::_xattr_rmattr( +SeaStore::Shard::tm_ret +SeaStore::Shard::_xattr_rmattr( internal_context_t &ctx, OnodeRef &onode, std::string &&name) @@ -1724,7 +1885,8 @@ SeaStore::tm_ret SeaStore::_xattr_rmattr( } } -SeaStore::tm_ret SeaStore::_rmattrs( +SeaStore::Shard::tm_ret +SeaStore::Shard::_rmattrs( internal_context_t &ctx, OnodeRef &onode) { @@ -1738,7 +1900,8 @@ SeaStore::tm_ret SeaStore::_rmattrs( return _xattr_clear(ctx, onode); } -SeaStore::tm_ret SeaStore::_xattr_clear( +SeaStore::Shard::tm_ret +SeaStore::Shard::_xattr_clear( internal_context_t &ctx, OnodeRef &onode) { @@ -1765,7 +1928,8 @@ SeaStore::tm_ret SeaStore::_xattr_clear( } } -SeaStore::tm_ret SeaStore::_create_collection( +SeaStore::Shard::tm_ret +SeaStore::Shard::_create_collection( internal_context_t &ctx, const coll_t& cid, int bits) { @@ -1797,7 +1961,8 @@ SeaStore::tm_ret SeaStore::_create_collection( ); } -SeaStore::tm_ret SeaStore::_remove_collection( +SeaStore::Shard::tm_ret +SeaStore::Shard::_remove_collection( internal_context_t &ctx, const coll_t& cid) { @@ -1828,13 +1993,15 @@ SeaStore::tm_ret SeaStore::_remove_collection( ); } -boost::intrusive_ptr SeaStore::_get_collection(const coll_t& cid) +boost::intrusive_ptr +SeaStore::Shard::_get_collection(const coll_t& cid) { return new SeastoreCollection{cid}; } -seastar::future<> SeaStore::write_meta(const std::string& key, - const std::string& value) +seastar::future<> SeaStore::Shard::write_meta( + const std::string& key, + const std::string& value) { LOG_PREFIX(SeaStore::write_meta); DEBUG("key: {}; value: {}", key, value); @@ -1854,16 +2021,16 @@ seastar::future<> SeaStore::write_meta(const std::string& key, return transaction_manager->submit_transaction(t); }); }); - }).safe_then([this, &key, &value] { - return mdstore->write_meta(key, value); }); }).handle_error( crimson::ct_error::assert_all{"Invalid error in SeaStore::write_meta"} ); } -seastar::future> SeaStore::read_meta(const std::string& key) +seastar::future> +SeaStore::read_meta(const std::string& key) { + ceph_assert(seastar::this_shard_id() == primary_core); LOG_PREFIX(SeaStore::read_meta); DEBUG("key: {}", key); return mdstore->read_meta(key).safe_then([](auto v) { @@ -1879,12 +2046,12 @@ seastar::future> SeaStore::read_meta(const std::str ); } -uuid_d SeaStore::get_fsid() const +uuid_d SeaStore::Shard::get_fsid() const { return device->get_meta().seastore_id; } -void SeaStore::init_managers() +void SeaStore::Shard::init_managers() { transaction_manager.reset(); collection_manager.reset(); @@ -1902,42 +2069,21 @@ void SeaStore::init_managers() *transaction_manager); } -seastar::future> make_seastore( - const std::string &device, - const ConfigValues &config) +std::unique_ptr make_seastore( + const std::string &device) { - using crimson::common::get_conf; - std::string type = get_conf("seastore_main_device_type"); - device_type_t d_type = string_to_device_type(type); - assert(d_type == device_type_t::SSD || - d_type == device_type_t::RANDOM_BLOCK_SSD); - - return Device::make_device( - device, d_type - ).then([&device](DeviceRef device_obj) { -#ifndef NDEBUG - bool is_test = true; -#else - bool is_test = false; -#endif - auto mdstore = std::make_unique(device); - return std::make_unique( - device, - std::move(mdstore), - std::move(device_obj), - is_test); - }); + auto mdstore = std::make_unique(device); + return std::make_unique( + device, + std::move(mdstore)); } std::unique_ptr make_test_seastore( - DeviceRef device, SeaStore::MDStoreRef mdstore) { return std::make_unique( "", - std::move(mdstore), - std::move(device), - true); + std::move(mdstore)); } } diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index 9ce77f7a826..8e43f275f18 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -33,6 +33,18 @@ class Onode; using OnodeRef = boost::intrusive_ptr; class TransactionManager; +enum class op_type_t : uint8_t { + TRANSACTION = 0, + READ, + WRITE, + GET_ATTR, + GET_ATTRS, + STAT, + OMAP_GET_VALUES, + OMAP_LIST, + MAX +}; + class SeastoreCollection final : public FuturizedCollection { public: template @@ -56,7 +68,6 @@ struct col_obj_ranges_t { ghobject_t obj_end; }; -using coll_core_t = FuturizedStore::coll_core_t; class SeaStore final : public FuturizedStore { public: class MDStore { @@ -80,159 +91,170 @@ public: }; using MDStoreRef = std::unique_ptr; - SeaStore( - const std::string& root, - MDStoreRef mdstore, - DeviceRef device, - bool is_test); - ~SeaStore(); - - seastar::future<> stop() final; - mount_ertr::future<> mount() final; - seastar::future<> umount() final; + class Shard : public FuturizedStore::Shard { + public: + Shard( + std::string root, + Device* device, + bool is_test); + ~Shard() = default; + + seastar::future stat( + CollectionRef c, + const ghobject_t& oid) final; + + read_errorator::future read( + CollectionRef c, + const ghobject_t& oid, + uint64_t offset, + size_t len, + uint32_t op_flags = 0) final; + + read_errorator::future readv( + CollectionRef c, + const ghobject_t& oid, + interval_set& m, + uint32_t op_flags = 0) final; + + get_attr_errorator::future get_attr( + CollectionRef c, + const ghobject_t& oid, + std::string_view name) const final; + + get_attrs_ertr::future get_attrs( + CollectionRef c, + const ghobject_t& oid) final; + + read_errorator::future omap_get_values( + CollectionRef c, + const ghobject_t& oid, + const omap_keys_t& keys) final; + + /// Retrieves paged set of values > start (if present) + using omap_get_values_ret_bare_t = std::tuple; + using omap_get_values_ret_t = read_errorator::future< + omap_get_values_ret_bare_t>; + omap_get_values_ret_t omap_get_values( + CollectionRef c, ///< [in] collection + const ghobject_t &oid, ///< [in] oid + const std::optional &start ///< [in] start, empty for begin + ) final; ///< @return values.empty() iff done + + get_attr_errorator::future omap_get_header( + CollectionRef c, + const ghobject_t& oid) final; + + seastar::future, ghobject_t>> list_objects( + CollectionRef c, + const ghobject_t& start, + const ghobject_t& end, + uint64_t limit) const final; + + seastar::future create_new_collection(const coll_t& cid) final; + seastar::future open_collection(const coll_t& cid) final; + + seastar::future<> do_transaction_no_callbacks( + CollectionRef ch, + ceph::os::Transaction&& txn) final; - mkfs_ertr::future<> mkfs(uuid_d new_osd_fsid) final; - seastar::future stat() const final; + /* Note, flush() machinery must go through the same pipeline + * stages and locks as do_transaction. */ + seastar::future<> flush(CollectionRef ch) final; - read_errorator::future read( - CollectionRef c, - const ghobject_t& oid, - uint64_t offset, - size_t len, - uint32_t op_flags = 0) final; - read_errorator::future readv( - CollectionRef c, - const ghobject_t& oid, - interval_set& m, - uint32_t op_flags = 0) final; - get_attr_errorator::future get_attr( - CollectionRef c, - const ghobject_t& oid, - std::string_view name) const final; - get_attrs_ertr::future get_attrs( - CollectionRef c, - const ghobject_t& oid) final; - - seastar::future stat( - CollectionRef c, - const ghobject_t& oid) final; - - read_errorator::future omap_get_values( - CollectionRef c, - const ghobject_t& oid, - const omap_keys_t& keys) final; - - /// Retrieves paged set of values > start (if present) - using omap_get_values_ret_bare_t = std::tuple; - using omap_get_values_ret_t = read_errorator::future< - omap_get_values_ret_bare_t>; - omap_get_values_ret_t omap_get_values( - CollectionRef c, ///< [in] collection - const ghobject_t &oid, ///< [in] oid - const std::optional &start ///< [in] start, empty for begin - ) final; ///< @return values.empty() iff done - - get_attr_errorator::future omap_get_header( - CollectionRef c, - const ghobject_t& oid) final; + read_errorator::future> fiemap( + CollectionRef ch, + const ghobject_t& oid, + uint64_t off, + uint64_t len) final; - static col_obj_ranges_t - get_objs_range(CollectionRef ch, unsigned bits); + unsigned get_max_attr_name_length() const final { + return 256; + } - seastar::future, ghobject_t>> list_objects( - CollectionRef c, - const ghobject_t& start, - const ghobject_t& end, - uint64_t limit) const final; + // only exposed to SeaStore + public: + mount_ertr::future<> mount(); - seastar::future create_new_collection(const coll_t& cid) final; - seastar::future open_collection(const coll_t& cid) final; - seastar::future> list_collections() final; + seastar::future<> umount(); - seastar::future<> do_transaction_no_callbacks( - CollectionRef ch, - ceph::os::Transaction&& txn) final; + seastar::future<> mkfs( + secondary_device_set_t &sds, + uuid_d new_osd_fsid); - /* Note, flush() machinery must go through the same pipeline - * stages and locks as do_transaction. */ - seastar::future<> flush(CollectionRef ch) final; + using coll_core_t = FuturizedStore::coll_core_t; + seastar::future> list_collections(); - read_errorator::future> fiemap( - CollectionRef ch, - const ghobject_t& oid, - uint64_t off, - uint64_t len) final; + seastar::future<> write_meta(const std::string& key, + const std::string& value); - seastar::future<> write_meta(const std::string& key, - const std::string& value) final; - seastar::future> read_meta(const std::string& key) final; - uuid_d get_fsid() const final; + store_statfs_t stat() const; - unsigned get_max_attr_name_length() const final { - return 256; - } - enum class op_type_t : uint8_t { - TRANSACTION = 0, - READ, - WRITE, - GET_ATTR, - GET_ATTRS, - STAT, - OMAP_GET_VALUES, - OMAP_LIST, - MAX - }; + uuid_d get_fsid() const; + // for each shard store make device + seastar::future<> make_shard_stores(); - // for test - mount_ertr::future<> test_mount(); - mkfs_ertr::future<> test_mkfs(uuid_d new_osd_fsid); - DeviceRef get_primary_device_ref() { - return std::move(device); - } + seastar::future<> mkfs_managers(); -private: - struct internal_context_t { - CollectionRef ch; - ceph::os::Transaction ext_transaction; + void init_managers(); - internal_context_t( - CollectionRef ch, - ceph::os::Transaction &&_ext_transaction, - TransactionRef &&transaction) - : ch(ch), ext_transaction(std::move(_ext_transaction)), - transaction(std::move(transaction)), - iter(ext_transaction.begin()) {} + TransactionManagerRef& get_transaction_manager() { + return transaction_manager; + } + // for secondaries device mkfs + seastar::future<> sec_mkfs( + const std::string path, + device_type_t dtype, + device_id_t id, + secondary_device_set_t &sds, + uuid_d new_osd_fsid); + + DeviceRef get_primary_device_ref() { + return std::move(device); + } - TransactionRef transaction; + private: + struct internal_context_t { + CollectionRef ch; + ceph::os::Transaction ext_transaction; - ceph::os::Transaction::iterator iter; - std::chrono::steady_clock::time_point begin_timestamp = std::chrono::steady_clock::now(); + internal_context_t( + CollectionRef ch, + ceph::os::Transaction &&_ext_transaction, + TransactionRef &&transaction) + : ch(ch), ext_transaction(std::move(_ext_transaction)), + transaction(std::move(transaction)), + iter(ext_transaction.begin()) {} - void reset_preserve_handle(TransactionManager &tm) { - tm.reset_transaction_preserve_handle(*transaction); - iter = ext_transaction.begin(); - } - }; + TransactionRef transaction; + + ceph::os::Transaction::iterator iter; + std::chrono::steady_clock::time_point begin_timestamp = std::chrono::steady_clock::now(); - TransactionManager::read_extent_iertr::future> - get_coll_bits(CollectionRef ch, Transaction &t) const; + void reset_preserve_handle(TransactionManager &tm) { + tm.reset_transaction_preserve_handle(*transaction); + iter = ext_transaction.begin(); + } + }; - static void on_error(ceph::os::Transaction &t); + TransactionManager::read_extent_iertr::future> + get_coll_bits(CollectionRef ch, Transaction &t) const; - template - auto repeat_with_internal_context( - CollectionRef ch, - ceph::os::Transaction &&t, - Transaction::src_t src, - const char* tname, - op_type_t op_type, - F &&f) { - return seastar::do_with( - internal_context_t( - ch, std::move(t), - transaction_manager->create_transaction(src, tname)), - std::forward(f), - [this, op_type](auto &ctx, auto &f) { + static void on_error(ceph::os::Transaction &t); + + template + auto repeat_with_internal_context( + CollectionRef ch, + ceph::os::Transaction &&t, + Transaction::src_t src, + const char* tname, + op_type_t op_type, + F &&f) { + return seastar::do_with( + internal_context_t( + ch, std::move(t), + transaction_manager->create_transaction(src, tname)), + std::forward(f), + [this, op_type](auto &ctx, auto &f) { return ctx.transaction->get_handle().take_collection_lock( static_cast(*(ctx.ch)).ordering_lock ).then([this] { @@ -253,206 +275,285 @@ private: }).finally([this] { throttler.put(); }); - } - ); - } + }); + } - template - auto repeat_with_onode( - CollectionRef ch, - const ghobject_t &oid, - Transaction::src_t src, - const char* tname, - op_type_t op_type, - F &&f) const { - auto begin_time = std::chrono::steady_clock::now(); - return seastar::do_with( - oid, Ret{}, std::forward(f), - [this, src, op_type, begin_time, tname - ](auto &oid, auto &ret, auto &f) - { - return repeat_eagain([&, this, src, tname] { - return transaction_manager->with_transaction_intr( - src, - tname, - [&, this](auto& t) - { - return onode_manager->get_onode(t, oid - ).si_then([&](auto onode) { - return seastar::do_with(std::move(onode), [&](auto& onode) { - return f(t, *onode); + template + auto repeat_with_onode( + CollectionRef ch, + const ghobject_t &oid, + Transaction::src_t src, + const char* tname, + op_type_t op_type, + F &&f) const { + auto begin_time = std::chrono::steady_clock::now(); + return seastar::do_with( + oid, Ret{}, std::forward(f), + [this, src, op_type, begin_time, tname + ](auto &oid, auto &ret, auto &f) + { + return repeat_eagain([&, this, src, tname] { + return transaction_manager->with_transaction_intr( + src, + tname, + [&, this](auto& t) + { + return onode_manager->get_onode(t, oid + ).si_then([&](auto onode) { + return seastar::do_with(std::move(onode), [&](auto& onode) { + return f(t, *onode); + }); + }).si_then([&ret](auto _ret) { + ret = _ret; }); - }).si_then([&ret](auto _ret) { - ret = _ret; }); + }).safe_then([&ret, op_type, begin_time, this] { + const_cast(this)->add_latency_sample(op_type, + std::chrono::steady_clock::now() - begin_time); + return seastar::make_ready_future(ret); }); - }).safe_then([&ret, op_type, begin_time, this] { - const_cast(this)->add_latency_sample(op_type, - std::chrono::steady_clock::now() - begin_time); - return seastar::make_ready_future(ret); + }); + } + + using _fiemap_ret = ObjectDataHandler::fiemap_ret; + _fiemap_ret _fiemap( + Transaction &t, + Onode &onode, + uint64_t off, + uint64_t len) const; + + using _omap_get_value_iertr = OMapManager::base_iertr::extend< + crimson::ct_error::enodata + >; + using _omap_get_value_ret = _omap_get_value_iertr::future; + _omap_get_value_ret _omap_get_value( + Transaction &t, + omap_root_t &&root, + std::string_view key) const; + + using _omap_get_values_iertr = OMapManager::base_iertr; + using _omap_get_values_ret = _omap_get_values_iertr::future; + _omap_get_values_ret _omap_get_values( + Transaction &t, + omap_root_t &&root, + const omap_keys_t &keys) const; + + friend class SeaStoreOmapIterator; + + using omap_list_bare_ret = OMapManager::omap_list_bare_ret; + using omap_list_ret = OMapManager::omap_list_ret; + omap_list_ret omap_list( + Onode &onode, + const omap_root_le_t& omap_root, + Transaction& t, + const std::optional& start, + OMapManager::omap_list_config_t config) const; + + using tm_iertr = TransactionManager::base_iertr; + using tm_ret = tm_iertr::future<>; + tm_ret _do_transaction_step( + internal_context_t &ctx, + CollectionRef &col, + std::vector &onodes, + std::vector &d_onodes, + ceph::os::Transaction::iterator &i); + + tm_ret _remove( + internal_context_t &ctx, + OnodeRef &onode); + tm_ret _touch( + internal_context_t &ctx, + OnodeRef &onode); + tm_ret _write( + internal_context_t &ctx, + OnodeRef &onode, + uint64_t offset, size_t len, + ceph::bufferlist &&bl, + uint32_t fadvise_flags); + tm_ret _zero( + internal_context_t &ctx, + OnodeRef &onode, + objaddr_t offset, extent_len_t len); + tm_ret _omap_set_values( + internal_context_t &ctx, + OnodeRef &onode, + std::map &&aset); + tm_ret _omap_set_header( + internal_context_t &ctx, + OnodeRef &onode, + ceph::bufferlist &&header); + tm_ret _omap_clear( + internal_context_t &ctx, + OnodeRef &onode); + tm_ret _omap_rmkeys( + internal_context_t &ctx, + OnodeRef &onode, + omap_keys_t &&aset); + tm_ret _omap_rmkeyrange( + internal_context_t &ctx, + OnodeRef &onode, + std::string first, + std::string last); + tm_ret _truncate( + internal_context_t &ctx, + OnodeRef &onode, uint64_t size); + tm_ret _setattrs( + internal_context_t &ctx, + OnodeRef &onode, + std::map&& aset); + tm_ret _rmattr( + internal_context_t &ctx, + OnodeRef &onode, + std::string name); + tm_ret _rmattrs( + internal_context_t &ctx, + OnodeRef &onode); + tm_ret _xattr_rmattr( + internal_context_t &ctx, + OnodeRef &onode, + std::string &&name); + tm_ret _xattr_clear( + internal_context_t &ctx, + OnodeRef &onode); + tm_ret _create_collection( + internal_context_t &ctx, + const coll_t& cid, int bits); + tm_ret _remove_collection( + internal_context_t &ctx, + const coll_t& cid); + using omap_set_kvs_ret = tm_iertr::future<>; + omap_set_kvs_ret _omap_set_kvs( + OnodeRef &onode, + const omap_root_le_t& omap_root, + Transaction& t, + omap_root_le_t& mutable_omap_root, + std::map&& kvs); + + boost::intrusive_ptr _get_collection(const coll_t& cid); + + static constexpr auto LAT_MAX = static_cast(op_type_t::MAX); + + struct { + std::array op_lat; + } stats; + + seastar::metrics::histogram& get_latency( + op_type_t op_type) { + assert(static_cast(op_type) < stats.op_lat.size()); + return stats.op_lat[static_cast(op_type)]; + } + + void add_latency_sample(op_type_t op_type, + std::chrono::steady_clock::duration dur) { + seastar::metrics::histogram& lat = get_latency(op_type); + lat.sample_count++; + lat.sample_sum += std::chrono::duration_cast(dur).count(); + } + + private: + std::string root; + DeviceRef device; + const uint32_t max_object_size; + bool is_test; + + std::vector secondaries; + TransactionManagerRef transaction_manager; + CollectionManagerRef collection_manager; + OnodeManagerRef onode_manager; + + common::Throttle throttler; + + seastar::metrics::metric_group metrics; + void register_metrics(); + }; + +public: + SeaStore( + const std::string& root, + MDStoreRef mdstore); + ~SeaStore(); + + seastar::future<> start() final; + seastar::future<> stop() final; + + mount_ertr::future<> mount() final { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.invoke_on_all( + [](auto &local_store) { + return local_store.mount().handle_error( + crimson::ct_error::assert_all{ + "Invalid error in SeaStore::mount" }); }); } - using _fiemap_ret = ObjectDataHandler::fiemap_ret; - _fiemap_ret _fiemap( - Transaction &t, - Onode &onode, - uint64_t off, - uint64_t len) const; + seastar::future<> umount() final { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.invoke_on_all( + [](auto &local_store) { + return local_store.umount(); + }); + } - using _omap_get_value_iertr = OMapManager::base_iertr::extend< - crimson::ct_error::enodata - >; - using _omap_get_value_ret = _omap_get_value_iertr::future; - _omap_get_value_ret _omap_get_value( - Transaction &t, - omap_root_t &&root, - std::string_view key) const; - - using _omap_get_values_iertr = OMapManager::base_iertr; - using _omap_get_values_ret = _omap_get_values_iertr::future; - _omap_get_values_ret _omap_get_values( - Transaction &t, - omap_root_t &&root, - const omap_keys_t &keys) const; - - friend class SeaStoreOmapIterator; - - using omap_list_bare_ret = OMapManager::omap_list_bare_ret; - using omap_list_ret = OMapManager::omap_list_ret; - omap_list_ret omap_list( - Onode &onode, - const omap_root_le_t& omap_root, - Transaction& t, - const std::optional& start, - OMapManager::omap_list_config_t config) const; - - void init_managers(); + mkfs_ertr::future<> mkfs(uuid_d new_osd_fsid) final; + seastar::future stat() const final; - std::string root; - MDStoreRef mdstore; - DeviceRef device; - const uint32_t max_object_size = 0; - bool is_test; - - std::vector secondaries; - TransactionManagerRef transaction_manager; - CollectionManagerRef collection_manager; - OnodeManagerRef onode_manager; - - common::Throttle throttler; - - using tm_iertr = TransactionManager::base_iertr; - using tm_ret = tm_iertr::future<>; - tm_ret _do_transaction_step( - internal_context_t &ctx, - CollectionRef &col, - std::vector &onodes, - std::vector &d_onodes, - ceph::os::Transaction::iterator &i); - - tm_ret _remove( - internal_context_t &ctx, - OnodeRef &onode); - tm_ret _touch( - internal_context_t &ctx, - OnodeRef &onode); - tm_ret _write( - internal_context_t &ctx, - OnodeRef &onode, - uint64_t offset, size_t len, - ceph::bufferlist &&bl, - uint32_t fadvise_flags); - tm_ret _zero( - internal_context_t &ctx, - OnodeRef &onode, - objaddr_t offset, extent_len_t len); - tm_ret _omap_set_values( - internal_context_t &ctx, - OnodeRef &onode, - std::map &&aset); - tm_ret _omap_set_header( - internal_context_t &ctx, - OnodeRef &onode, - ceph::bufferlist &&header); - tm_ret _omap_clear( - internal_context_t &ctx, - OnodeRef &onode); - tm_ret _omap_rmkeys( - internal_context_t &ctx, - OnodeRef &onode, - omap_keys_t &&aset); - tm_ret _omap_rmkeyrange( - internal_context_t &ctx, - OnodeRef &onode, - std::string first, - std::string last); - tm_ret _truncate( - internal_context_t &ctx, - OnodeRef &onode, uint64_t size); - tm_ret _setattrs( - internal_context_t &ctx, - OnodeRef &onode, - std::map&& aset); - tm_ret _rmattr( - internal_context_t &ctx, - OnodeRef &onode, - std::string name); - tm_ret _rmattrs( - internal_context_t &ctx, - OnodeRef &onode); - tm_ret _xattr_rmattr( - internal_context_t &ctx, - OnodeRef &onode, - std::string &&name); - tm_ret _xattr_clear( - internal_context_t &ctx, - OnodeRef &onode); - tm_ret _create_collection( - internal_context_t &ctx, - const coll_t& cid, int bits); - tm_ret _remove_collection( - internal_context_t &ctx, - const coll_t& cid); - using omap_set_kvs_ret = tm_iertr::future<>; - omap_set_kvs_ret _omap_set_kvs( - OnodeRef &onode, - const omap_root_le_t& omap_root, - Transaction& t, - omap_root_le_t& mutable_omap_root, - std::map&& kvs); - - boost::intrusive_ptr _get_collection(const coll_t& cid); - - static constexpr auto LAT_MAX = static_cast(op_type_t::MAX); - struct { - std::array op_lat; - } stats; - - seastar::metrics::histogram& get_latency( - op_type_t op_type) { - assert(static_cast(op_type) < stats.op_lat.size()); - return stats.op_lat[static_cast(op_type)]; + uuid_d get_fsid() const final { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.local().get_fsid(); + } + + seastar::future<> write_meta( + const std::string& key, + const std::string& value) final { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.local().write_meta( + key, value).then([this, key, value] { + return mdstore->write_meta(key, value); + }).handle_error( + crimson::ct_error::assert_all{"Invalid error in SeaStore::write_meta"} + ); + } + + seastar::future> read_meta(const std::string& key) final; + + seastar::future> list_collections() final; + + FuturizedStore::Shard& get_sharded_store() final { + return shard_stores.local(); } - void add_latency_sample(op_type_t op_type, - std::chrono::steady_clock::duration dur) { - seastar::metrics::histogram& lat = get_latency(op_type); - lat.sample_count++; - lat.sample_sum += std::chrono::duration_cast(dur).count(); + static col_obj_ranges_t + get_objs_range(CollectionRef ch, unsigned bits); + +// for test +public: + mount_ertr::future<> test_mount(); + mkfs_ertr::future<> test_mkfs(uuid_d new_osd_fsid); + + DeviceRef get_primary_device_ref() { + ceph_assert(seastar::this_shard_id() == primary_core); + return shard_stores.local().get_primary_device_ref(); } - seastar::metrics::metric_group metrics; - void register_metrics(); + + seastar::future<> test_start(DeviceRef dev); + +private: seastar::future<> write_fsid(uuid_d new_osd_fsid); + + seastar::future<> prepare_meta(uuid_d new_osd_fsid); + seastar::future<> _mkfs(uuid_d new_osd_fsid); + +private: + std::string root; + MDStoreRef mdstore; + seastar::sharded shard_stores; }; -seastar::future> make_seastore( - const std::string &device, - const ConfigValues &config); +std::unique_ptr make_seastore( + const std::string &device); std::unique_ptr make_test_seastore( - DeviceRef device, SeaStore::MDStoreRef mdstore); } diff --git a/src/crimson/os/seastore/segment_manager.cc b/src/crimson/os/seastore/segment_manager.cc index fe0967e455e..44192965fce 100644 --- a/src/crimson/os/seastore/segment_manager.cc +++ b/src/crimson/os/seastore/segment_manager.cc @@ -52,7 +52,7 @@ LOG_PREFIX(SegmentManager::get_segment_manager); static_cast(0), [&](auto &nr_zones) { return seastar::open_file_dma( - device + "/block", + device + "/block" + std::to_string(seastar::this_shard_id()), seastar::open_flags::rw ).then([&](auto file) { return seastar::do_with( @@ -67,11 +67,11 @@ LOG_PREFIX(SegmentManager::get_segment_manager); if (nr_zones != 0) { return std::make_unique< segment_manager::zns::ZNSSegmentManager - >(device + "/block"); + >(device + "/block" + std::to_string(seastar::this_shard_id())); } else { return std::make_unique< segment_manager::block::BlockSegmentManager - >(device + "/block", dtype); + >(device + "/block" + std::to_string(seastar::this_shard_id()), dtype); } }); }); @@ -79,7 +79,7 @@ LOG_PREFIX(SegmentManager::get_segment_manager); return seastar::make_ready_future( std::make_unique< segment_manager::block::BlockSegmentManager - >(device + "/block", dtype)); + >(device + "/block" + std::to_string(seastar::this_shard_id()), dtype)); #endif } diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index 04b83b56c5f..8ed119cb9f7 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -206,7 +206,7 @@ block_sm_superblock_t make_superblock( using crimson::common::get_conf; auto config_size = get_conf( - "seastore_device_size"); + "seastore_device_size")/seastar::smp::count; size_t size = (data.size == 0) ? config_size : data.size; @@ -530,7 +530,8 @@ BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs( check_create_device_ret maybe_create = check_create_device_ertr::now(); using crimson::common::get_conf; if (get_conf("seastore_block_create")) { - auto size = get_conf("seastore_device_size"); + auto size = + get_conf("seastore_device_size")/seastar::smp::count; maybe_create = check_create_device(device_path, size); } diff --git a/src/crimson/osd/ec_backend.h b/src/crimson/osd/ec_backend.h index 74e3cd8919b..3dbcc4def2e 100644 --- a/src/crimson/osd/ec_backend.h +++ b/src/crimson/osd/ec_backend.h @@ -33,7 +33,7 @@ private: epoch_t min_epoch, epoch_t max_epoch, std::vector&& log_entries) final; CollectionRef coll; - crimson::os::FuturizedStore* store; + crimson::os::FuturizedStore::Shard* store; seastar::future<> request_committed(const osd_reqid_t& reqid, const eversion_t& version) final { return seastar::now(); diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc index 3bfe9893b95..234259f0e6d 100644 --- a/src/crimson/osd/main.cc +++ b/src/crimson/osd/main.cc @@ -197,7 +197,7 @@ int main(int argc, const char* argv[]) auto store = crimson::os::FuturizedStore::create( local_conf().get_val("osd_objectstore"), local_conf().get_val("osd_data"), - local_conf().get_config_values()).get(); + local_conf().get_config_values()); crimson::osd::OSD osd( whoami, nonce, std::ref(should_stop.abort_source()), diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 6ec72bf6933..035b40f4215 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -158,25 +158,25 @@ CompatSet get_osd_initial_compat_set() seastar::future<> OSD::open_meta_coll() { - return store.open_collection( + return store.get_sharded_store().open_collection( coll_t::meta() ).then([this](auto ch) { - pg_shard_manager.init_meta_coll(ch, store); + pg_shard_manager.init_meta_coll(ch, store.get_sharded_store()); return seastar::now(); }); } seastar::future OSD::open_or_create_meta_coll(FuturizedStore &store) { - return store.open_collection(coll_t::meta()).then([&store](auto ch) { + return store.get_sharded_store().open_collection(coll_t::meta()).then([&store](auto ch) { if (!ch) { - return store.create_new_collection( + return store.get_sharded_store().create_new_collection( coll_t::meta() ).then([&store](auto ch) { - return OSDMeta(ch, store); + return OSDMeta(ch, store.get_sharded_store()); }); } else { - return seastar::make_ready_future(ch, store); + return seastar::make_ready_future(ch, store.get_sharded_store()); } }); } @@ -269,7 +269,7 @@ seastar::future<> OSD::_write_superblock( meta_coll.create(t); meta_coll.store_superblock(t, superblock); logger().debug("OSD::_write_superblock: do_transaction..."); - return store.do_transaction( + return store.get_sharded_store().do_transaction( meta_coll.collection(), std::move(t)); }), @@ -354,15 +354,14 @@ seastar::future<> OSD::start() startup_time = ceph::mono_clock::now(); - return pg_shard_manager.start( + return store.start().then([this] { + return pg_shard_manager.start( whoami, *cluster_msgr, - *public_msgr, *monc, *mgrc, store - ).then([this] { + *public_msgr, *monc, *mgrc, store); + }).then([this] { heartbeat.reset(new Heartbeat{ whoami, get_shard_services(), *monc, *hb_front_msgr, *hb_back_msgr}); - return store.start(); - }).then([this] { return store.mount().handle_error( crimson::stateful_ec::handle([] (const auto& ec) { logger().error("error mounting object store in {}: ({}) {}", @@ -387,7 +386,7 @@ seastar::future<> OSD::start() }).then([this] { pg_shard_manager.got_map(osdmap->get_epoch()); bind_epoch = osdmap->get_epoch(); - return pg_shard_manager.load_pgs(); + return pg_shard_manager.load_pgs(store); }).then([this] { uint64_t osd_required = @@ -910,7 +909,7 @@ seastar::future<> OSD::handle_osd_map(crimson::net::ConnectionRef conn, pg_shard_manager.get_meta_coll().store_superblock(t, superblock); pg_shard_manager.set_superblock(superblock); logger().debug("OSD::handle_osd_map: do_transaction..."); - return store.do_transaction( + return store.get_sharded_store().do_transaction( pg_shard_manager.get_meta_coll().collection(), std::move(t)); }); diff --git a/src/crimson/osd/osd_meta.cc b/src/crimson/osd/osd_meta.cc index 973169fa4af..aa30b8190fd 100644 --- a/src/crimson/osd/osd_meta.cc +++ b/src/crimson/osd/osd_meta.cc @@ -11,7 +11,7 @@ #include "os/Transaction.h" using std::string; -using read_errorator = crimson::os::FuturizedStore::read_errorator; +using read_errorator = crimson::os::FuturizedStore::Shard::read_errorator; void OSDMeta::create(ceph::os::Transaction& t) { diff --git a/src/crimson/osd/osd_meta.h b/src/crimson/osd/osd_meta.h index 716af2eb232..652266d9e20 100644 --- a/src/crimson/osd/osd_meta.h +++ b/src/crimson/osd/osd_meta.h @@ -24,12 +24,12 @@ namespace crimson::os { class OSDMeta { template using Ref = boost::intrusive_ptr; - crimson::os::FuturizedStore& store; + crimson::os::FuturizedStore::Shard& store; Ref coll; public: OSDMeta(Ref coll, - crimson::os::FuturizedStore& store) + crimson::os::FuturizedStore::Shard& store) : store{store}, coll{coll} {} @@ -45,7 +45,7 @@ public: void store_superblock(ceph::os::Transaction& t, const OSDSuperblock& sb); - using load_superblock_ertr = crimson::os::FuturizedStore::read_errorator; + using load_superblock_ertr = crimson::os::FuturizedStore::Shard::read_errorator; using load_superblock_ret = load_superblock_ertr::future; load_superblock_ret load_superblock(); diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc index 997ad0953fd..c38bf8be3a1 100644 --- a/src/crimson/osd/pg.cc +++ b/src/crimson/osd/pg.cc @@ -612,7 +612,7 @@ void PG::init( new_acting_primary, history, pi, t); } -seastar::future<> PG::read_state(crimson::os::FuturizedStore* store) +seastar::future<> PG::read_state(crimson::os::FuturizedStore::Shard* store) { if (__builtin_expect(stopping, false)) { return seastar::make_exception_future<>( diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h index aecd2950875..4459d5fe496 100644 --- a/src/crimson/osd/pg.h +++ b/src/crimson/osd/pg.h @@ -491,7 +491,7 @@ public: const PastIntervals& pim, ceph::os::Transaction &t); - seastar::future<> read_state(crimson::os::FuturizedStore* store); + seastar::future<> read_state(crimson::os::FuturizedStore::Shard* store); interruptible_future<> do_peering_event( PGPeeringEvent& evt, PeeringCtx &rctx); diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc index 4c6b7fb33a9..d69e5e2042e 100644 --- a/src/crimson/osd/pg_backend.cc +++ b/src/crimson/osd/pg_backend.cc @@ -1280,7 +1280,7 @@ void PGBackend::clone( } using get_omap_ertr = - crimson::os::FuturizedStore::read_errorator::extend< + crimson::os::FuturizedStore::Shard::read_errorator::extend< crimson::ct_error::enodata>; using get_omap_iertr = ::crimson::interruptible::interruptible_errorator< @@ -1288,9 +1288,9 @@ using get_omap_iertr = get_omap_ertr>; static get_omap_iertr::future< - crimson::os::FuturizedStore::omap_values_t> + crimson::os::FuturizedStore::Shard::omap_values_t> maybe_get_omap_vals_by_keys( - crimson::os::FuturizedStore* store, + crimson::os::FuturizedStore::Shard* store, const crimson::os::CollectionRef& coll, const object_info_t& oi, const std::set& keys_to_get) @@ -1304,9 +1304,9 @@ maybe_get_omap_vals_by_keys( static get_omap_iertr::future< - std::tuple> + std::tuple> maybe_get_omap_vals( - crimson::os::FuturizedStore* store, + crimson::os::FuturizedStore::Shard* store, const crimson::os::CollectionRef& coll, const object_info_t& oi, const std::string& start_after) @@ -1557,7 +1557,7 @@ PGBackend::omap_get_vals_by_keys( delta_stats.num_rd++; return maybe_get_omap_vals_by_keys(store, coll, os.oi, keys_to_get) .safe_then_interruptible( - [&osd_op] (crimson::os::FuturizedStore::omap_values_t&& vals) { + [&osd_op] (crimson::os::FuturizedStore::Shard::omap_values_t&& vals) { encode(vals, osd_op.outdata); return ll_read_errorator::now(); }).handle_error_interruptible( diff --git a/src/crimson/osd/pg_backend.h b/src/crimson/osd/pg_backend.h index aa581134420..fbad37d4c71 100644 --- a/src/crimson/osd/pg_backend.h +++ b/src/crimson/osd/pg_backend.h @@ -40,7 +40,7 @@ protected: using CollectionRef = crimson::os::CollectionRef; using ec_profile_t = std::map; // low-level read errorator - using ll_read_errorator = crimson::os::FuturizedStore::read_errorator; + using ll_read_errorator = crimson::os::FuturizedStore::Shard::read_errorator; using ll_read_ierrorator = ::crimson::interruptible::interruptible_errorator< ::crimson::osd::IOInterruptCondition, @@ -238,7 +238,7 @@ public: const OSDOp& osd_op, ceph::os::Transaction& trans, object_stat_sum_t& delta_stats); - using get_attr_errorator = crimson::os::FuturizedStore::get_attr_errorator; + using get_attr_errorator = crimson::os::FuturizedStore::Shard::get_attr_errorator; using get_attr_ierrorator = ::crimson::interruptible::interruptible_errorator< ::crimson::osd::IOInterruptCondition, @@ -322,7 +322,7 @@ public: OSDOp& osd_op, object_stat_sum_t& delta_stats) const; using omap_cmp_ertr = - crimson::os::FuturizedStore::read_errorator::extend< + crimson::os::FuturizedStore::Shard::read_errorator::extend< crimson::ct_error::ecanceled, crimson::ct_error::invarg>; using omap_cmp_iertr = @@ -389,7 +389,7 @@ protected: CollectionRef coll; crimson::osd::ShardServices &shard_services; DoutPrefixProvider &dpp; ///< provides log prefix context - crimson::os::FuturizedStore* store; + crimson::os::FuturizedStore::Shard* store; virtual seastar::future<> request_committed( const osd_reqid_t& reqid, const eversion_t& at_version) = 0; diff --git a/src/crimson/osd/pg_meta.cc b/src/crimson/osd/pg_meta.cc index a0b9b37d603..288ee52a086 100644 --- a/src/crimson/osd/pg_meta.cc +++ b/src/crimson/osd/pg_meta.cc @@ -14,14 +14,14 @@ using std::string_view; // easily skip them using crimson::os::FuturizedStore; -PGMeta::PGMeta(FuturizedStore& store, spg_t pgid) +PGMeta::PGMeta(FuturizedStore::Shard& store, spg_t pgid) : store{store}, pgid{pgid} {} namespace { template - std::optional find_value(const FuturizedStore::omap_values_t& values, + std::optional find_value(const FuturizedStore::Shard::omap_values_t& values, string_view key) { auto found = values.find(key); @@ -57,7 +57,7 @@ seastar::future PGMeta::get_epoch() return seastar::make_ready_future(*epoch); } }, - FuturizedStore::read_errorator::assert_all{ + FuturizedStore::Shard::read_errorator::assert_all{ "PGMeta::get_epoch: unable to read pgmeta" }); }); @@ -104,7 +104,7 @@ seastar::future> PGMeta::load() return seastar::make_ready_future>( std::make_tuple(std::move(info), std::move(past_intervals))); }, - FuturizedStore::read_errorator::assert_all{ + FuturizedStore::Shard::read_errorator::assert_all{ "PGMeta::load: unable to read pgmeta" }); } diff --git a/src/crimson/osd/pg_meta.h b/src/crimson/osd/pg_meta.h index 3b1ab92c387..21c2bb373b1 100644 --- a/src/crimson/osd/pg_meta.h +++ b/src/crimson/osd/pg_meta.h @@ -6,18 +6,15 @@ #include #include #include "osd/osd_types.h" - -namespace crimson::os { - class FuturizedStore; -} +#include "crimson/os/futurized_store.h" /// PG related metadata class PGMeta { - crimson::os::FuturizedStore& store; + crimson::os::FuturizedStore::Shard& store; const spg_t pgid; public: - PGMeta(crimson::os::FuturizedStore& store, spg_t pgid); + PGMeta(crimson::os::FuturizedStore::Shard& store, spg_t pgid); seastar::future get_epoch(); seastar::future> load(); }; diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc index c81ea8a01f4..03174b1df05 100644 --- a/src/crimson/osd/pg_shard_manager.cc +++ b/src/crimson/osd/pg_shard_manager.cc @@ -45,10 +45,10 @@ seastar::future<> PGShardManager::stop() }); } -seastar::future<> PGShardManager::load_pgs() +seastar::future<> PGShardManager::load_pgs(crimson::os::FuturizedStore& store) { ceph_assert(seastar::this_shard_id() == PRIMARY_CORE); - return get_local_state().store.list_collections( + return store.list_collections( ).then([this](auto colls_cores) { return seastar::parallel_for_each( colls_cores, diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h index fc56e64eb1b..9856edfb6de 100644 --- a/src/crimson/osd/pg_shard_manager.h +++ b/src/crimson/osd/pg_shard_manager.h @@ -10,8 +10,11 @@ #include "crimson/osd/shard_services.h" #include "crimson/osd/pg_map.h" -namespace crimson::osd { +namespace crimson::os { + class FuturizedStore; +} +namespace crimson::osd { /** * PGShardManager * @@ -233,7 +236,7 @@ public: }); } - seastar::future<> load_pgs(); + seastar::future<> load_pgs(crimson::os::FuturizedStore& store); seastar::future<> stop_pgs(); seastar::future> get_pg_stats() const; diff --git a/src/crimson/osd/recovery_backend.h b/src/crimson/osd/recovery_backend.h index 64be1862074..24b674dc815 100644 --- a/src/crimson/osd/recovery_backend.h +++ b/src/crimson/osd/recovery_backend.h @@ -95,7 +95,7 @@ public: protected: crimson::osd::PG& pg; crimson::osd::ShardServices& shard_services; - crimson::os::FuturizedStore* store; + crimson::os::FuturizedStore::Shard* store; crimson::os::CollectionRef coll; PGBackend* backend; diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc index b2e3dfbd638..60da07a9e9b 100644 --- a/src/crimson/osd/replicated_recovery_backend.cc +++ b/src/crimson/osd/replicated_recovery_backend.cc @@ -455,17 +455,17 @@ ReplicatedRecoveryBackend::read_metadata_for_push_op( } return interruptor::make_interruptible(interruptor::when_all_succeed( backend->omap_get_header(coll, ghobject_t(oid)).handle_error_interruptible( - crimson::os::FuturizedStore::read_errorator::all_same_way( + crimson::os::FuturizedStore::Shard::read_errorator::all_same_way( [oid] (const std::error_code& e) { logger().debug("read_metadata_for_push_op, error {} when getting omap header: {}", e, oid); return seastar::make_ready_future(); })), interruptor::make_interruptible(store->get_attrs(coll, ghobject_t(oid))) .handle_error_interruptible( - crimson::os::FuturizedStore::get_attrs_ertr::all_same_way( + crimson::os::FuturizedStore::Shard::get_attrs_ertr::all_same_way( [oid] (const std::error_code& e) { logger().debug("read_metadata_for_push_op, error {} when getting attrs: {}", e, oid); - return seastar::make_ready_future(); + return seastar::make_ready_future(); })) )).then_unpack_interruptible([&new_progress, push_op](auto bl, auto attrs) { if (bl.length() == 0) { @@ -593,7 +593,7 @@ ReplicatedRecoveryBackend::read_omap_for_push_op( return seastar::make_ready_future( stop ? seastar::stop_iteration::yes : seastar::stop_iteration::no ); - }, crimson::os::FuturizedStore::read_errorator::assert_all{}); + }, crimson::os::FuturizedStore::Shard::read_errorator::assert_all{}); }); } diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc index 489eac36a58..647d8d6bee4 100644 --- a/src/crimson/osd/shard_services.cc +++ b/src/crimson/osd/shard_services.cc @@ -40,7 +40,7 @@ PerShardState::PerShardState( PerfCounters *recoverystate_perf, crimson::os::FuturizedStore &store) : whoami(whoami), - store(store), + store(store.get_sharded_store()), perf(perf), recoverystate_perf(recoverystate_perf), throttler(crimson::common::local_conf()), next_tid( diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h index 5f6c57c41b1..12bb23ac2ab 100644 --- a/src/crimson/osd/shard_services.h +++ b/src/crimson/osd/shard_services.h @@ -65,7 +65,7 @@ class PerShardState { #define assert_core() ceph_assert(seastar::this_shard_id() == core); const int whoami; - crimson::os::FuturizedStore &store; + crimson::os::FuturizedStore::Shard &store; crimson::common::CephContext cct; PerfCounters *perf = nullptr; @@ -374,7 +374,7 @@ public: FORWARD_TO_OSD_SINGLETON(send_to_osd) - crimson::os::FuturizedStore &get_store() { + crimson::os::FuturizedStore::Shard &get_store() { return local_state.store; } diff --git a/src/crimson/tools/store_nbd/fs_driver.cc b/src/crimson/tools/store_nbd/fs_driver.cc index f7a33b38510..18f83676645 100644 --- a/src/crimson/tools/store_nbd/fs_driver.cc +++ b/src/crimson/tools/store_nbd/fs_driver.cc @@ -145,7 +145,7 @@ seastar::future<> FSDriver::write( config.log_size); } - return fs->do_transaction( + return sharded_fs->do_transaction( mapping.pg.collection, std::move(t)); } @@ -156,7 +156,7 @@ seastar::future FSDriver::read( { auto mapping = map_offset(offset); ceph_assert((mapping.offset + size) <= config.object_size); - return fs->read( + return sharded_fs->read( mapping.pg.collection, mapping.object, mapping.offset, @@ -179,11 +179,9 @@ seastar::future FSDriver::read( seastar::future<> FSDriver::mkfs() { - return init( + return init( ).then([this] { assert(fs); - return fs->start(); - }).then([this] { uuid_d uuid; uuid.generate_random(); return fs->mkfs(uuid).handle_error( @@ -197,9 +195,7 @@ seastar::future<> FSDriver::mkfs() }).then([this] { return fs->stop(); }).then([this] { - return init().then([this] { - return fs->start(); - }); + return init(); }).then([this] { return fs->mount( ).handle_error( @@ -218,11 +214,11 @@ seastar::future<> FSDriver::mkfs() boost::counting_iterator(0), boost::counting_iterator(config.num_pgs), [this](auto i) { - return fs->create_new_collection(get_coll(i) + return sharded_fs->create_new_collection(get_coll(i) ).then([this, i](auto coll) { ceph::os::Transaction t; t.create_collection(get_coll(i), 0); - return fs->do_transaction(coll, std::move(t)); + return sharded_fs->do_transaction(coll, std::move(t)); }); }); }).then([this] { @@ -241,9 +237,7 @@ seastar::future<> FSDriver::mount() return ( config.mkfs ? mkfs() : seastar::now() ).then([this] { - return init().then([this] { - return fs->start(); - }); + return init(); }).then([this] { return fs->mount( ).handle_error( @@ -262,7 +256,7 @@ seastar::future<> FSDriver::mount() boost::counting_iterator(0), boost::counting_iterator(config.num_pgs), [this](auto i) { - return fs->open_collection(get_coll(i) + return sharded_fs->open_collection(get_coll(i) ).then([this, i](auto ref) { collections[i].collection = ref; collections[i].log_object = get_log_object(i); @@ -275,7 +269,7 @@ seastar::future<> FSDriver::mount() config.log_entry_size, config.log_size); } - return fs->do_transaction( + return sharded_fs->do_transaction( collections[i].collection, std::move(t)); } else { @@ -305,12 +299,12 @@ seastar::future<> FSDriver::close() seastar::future<> FSDriver::init() { fs.reset(); - return FuturizedStore::create( + fs = FuturizedStore::create( config.get_fs_type(), *config.path, crimson::common::local_conf().get_config_values() - ).then([this] (auto store_ptr) { - fs = std::move(store_ptr); - return seastar::now(); + ); + return fs->start().then([this] { + sharded_fs = &(fs->get_sharded_store()); }); } diff --git a/src/crimson/tools/store_nbd/fs_driver.h b/src/crimson/tools/store_nbd/fs_driver.h index 87af3dd129b..89aca075f8a 100644 --- a/src/crimson/tools/store_nbd/fs_driver.h +++ b/src/crimson/tools/store_nbd/fs_driver.h @@ -37,6 +37,7 @@ private: size_t size = 0; const config_t config; std::unique_ptr fs; + crimson::os::FuturizedStore::Shard* sharded_fs; struct pg_analogue_t { crimson::os::CollectionRef collection; diff --git a/src/osd/PGLog.cc b/src/osd/PGLog.cc index fdb3715e913..11603669377 100644 --- a/src/osd/PGLog.cc +++ b/src/osd/PGLog.cc @@ -1074,8 +1074,8 @@ void PGLog::rebuild_missing_set_with_deletes( #ifdef WITH_SEASTAR namespace { - struct FuturizedStoreLogReader { - crimson::os::FuturizedStore &store; + struct FuturizedShardStoreLogReader { + crimson::os::FuturizedStore::Shard &store; const pg_info_t &info; PGLog::IndexedLog &log; std::set* log_keys_debug = NULL; @@ -1165,7 +1165,7 @@ namespace { return seastar::make_ready_future( done ? seastar::stop_iteration::yes : seastar::stop_iteration::no ); - }, crimson::os::FuturizedStore::read_errorator::assert_all{}); + }, crimson::os::FuturizedStore::Shard::read_errorator::assert_all{}); }).then([this] { if (info.pgid.is_no_shard()) { // replicated pool pg does not persist this key @@ -1186,7 +1186,7 @@ namespace { } seastar::future<> PGLog::read_log_and_missing_crimson( - crimson::os::FuturizedStore &store, + crimson::os::FuturizedStore::Shard &store, crimson::os::CollectionRef ch, const pg_info_t &info, IndexedLog &log, @@ -1198,16 +1198,16 @@ seastar::future<> PGLog::read_log_and_missing_crimson( ldpp_dout(dpp, 20) << "read_log_and_missing coll " << ch->get_cid() << " " << pgmeta_oid << dendl; - return seastar::do_with(FuturizedStoreLogReader{ + return seastar::do_with(FuturizedShardStoreLogReader{ store, info, log, log_keys_debug, missing, dpp}, - [ch, pgmeta_oid](FuturizedStoreLogReader& reader) { + [ch, pgmeta_oid](FuturizedShardStoreLogReader& reader) { return reader.read(ch, pgmeta_oid); }); } seastar::future<> PGLog::rebuild_missing_set_with_deletes_crimson( - crimson::os::FuturizedStore &store, + crimson::os::FuturizedStore::Shard &store, crimson::os::CollectionRef ch, const pg_info_t &info) { diff --git a/src/osd/PGLog.h b/src/osd/PGLog.h index 10016752bf7..a9281e1b5f9 100644 --- a/src/osd/PGLog.h +++ b/src/osd/PGLog.h @@ -945,7 +945,7 @@ public: #ifdef WITH_SEASTAR seastar::future<> rebuild_missing_set_with_deletes_crimson( - crimson::os::FuturizedStore &store, + crimson::os::FuturizedStore::Shard &store, crimson::os::CollectionRef ch, const pg_info_t &info); #endif @@ -1683,7 +1683,7 @@ public: #ifdef WITH_SEASTAR seastar::future<> read_log_and_missing_crimson( - crimson::os::FuturizedStore &store, + crimson::os::FuturizedStore::Shard &store, crimson::os::CollectionRef ch, const pg_info_t &info, ghobject_t pgmeta_oid @@ -1695,7 +1695,7 @@ public: } static seastar::future<> read_log_and_missing_crimson( - crimson::os::FuturizedStore &store, + crimson::os::FuturizedStore::Shard &store, crimson::os::CollectionRef ch, const pg_info_t &info, IndexedLog &log, diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc index 7871fec2e8e..7893bc08fdc 100644 --- a/src/osd/SnapMapper.cc +++ b/src/osd/SnapMapper.cc @@ -99,11 +99,11 @@ int OSDriver::get_keys( using crimson::os::FuturizedStore; return interruptor::green_get(os->omap_get_values( ch, hoid, keys - ).safe_then([out] (FuturizedStore::omap_values_t&& vals) { + ).safe_then([out] (FuturizedStore::Shard::omap_values_t&& vals) { // just the difference in comparator (`std::less<>` in omap_values_t`) - reinterpret_cast(*out) = std::move(vals); + reinterpret_cast(*out) = std::move(vals); return 0; - }, FuturizedStore::read_errorator::all_same_way([] (auto& e) { + }, FuturizedStore::Shard::read_errorator::all_same_way([] (auto& e) { assert(e.value() > 0); return -e.value(); }))); // this requires seastar::thread @@ -118,7 +118,7 @@ int OSDriver::get_next( using crimson::os::FuturizedStore; return interruptor::green_get(os->omap_get_values( ch, hoid, key - ).safe_then_unpack([&key, next] (bool, FuturizedStore::omap_values_t&& vals) { + ).safe_then_unpack([&key, next] (bool, FuturizedStore::Shard::omap_values_t&& vals) { CRIMSON_DEBUG("OSDriver::{}:{}", "get_next", __LINE__); if (auto nit = std::begin(vals); nit == std::end(vals)) { CRIMSON_DEBUG("OSDriver::{}:{}", "get_next", __LINE__); @@ -129,7 +129,7 @@ int OSDriver::get_next( *next = *nit; return 0; } - }, FuturizedStore::read_errorator::all_same_way([] { + }, FuturizedStore::Shard::read_errorator::all_same_way([] { CRIMSON_DEBUG("OSDriver::{}:{}", "get_next", __LINE__); return -EINVAL; }))); // this requires seastar::thread @@ -144,12 +144,12 @@ int OSDriver::get_next_or_current( using crimson::os::FuturizedStore; // let's try to get current first return interruptor::green_get(os->omap_get_values( - ch, hoid, FuturizedStore::omap_keys_t{key} - ).safe_then([&key, next_or_current] (FuturizedStore::omap_values_t&& vals) { + ch, hoid, FuturizedStore::Shard::omap_keys_t{key} + ).safe_then([&key, next_or_current] (FuturizedStore::Shard::omap_values_t&& vals) { assert(vals.size() == 1); *next_or_current = std::make_pair(key, std::move(vals[0])); return 0; - }, FuturizedStore::read_errorator::all_same_way( + }, FuturizedStore::Shard::read_errorator::all_same_way( [next_or_current, &key, this] { // no current, try next return get_next(key, next_or_current); diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h index 95c9a6bb8fe..eb43a23c2b0 100644 --- a/src/osd/SnapMapper.h +++ b/src/osd/SnapMapper.h @@ -35,7 +35,7 @@ class OSDriver : public MapCacher::StoreDriver { #ifdef WITH_SEASTAR - using ObjectStoreT = crimson::os::FuturizedStore; + using ObjectStoreT = crimson::os::FuturizedStore::Shard; using CollectionHandleT = ObjectStoreT::CollectionRef; #else using ObjectStoreT = ObjectStore; diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 58508ac594a..ae0953b4541 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -179,7 +179,7 @@ protected: num_main_device_managers(num_main_device_managers), num_cold_device_managers(num_cold_device_managers) {} - virtual void _init() = 0; + virtual seastar::future<> _init() = 0; virtual void _destroy() = 0; virtual seastar::future<> _teardown() = 0; @@ -197,8 +197,9 @@ protected: SUBINFO(test, "begin ..."); return teardown().then([this] { devices->remount(); - _init(); - return _mount().handle_error(crimson::ct_error::assert_all{}); + return _init().then([this] { + return _mount().handle_error(crimson::ct_error::assert_all{}); + }); }).then([FNAME] { SUBINFO(test, "finish"); }); @@ -225,16 +226,17 @@ protected: } SUBINFO(test, "begin with {} devices ...", devices->get_num_devices()); return devices->setup( - ).then([this, FNAME]() { - _init(); - return _mkfs( + ).then([this]() { + return _init(); + }).then([this, FNAME] { + return _mkfs( ).safe_then([this] { - return restart_fut(); + return restart_fut(); }).handle_error( - crimson::ct_error::assert_all{} + crimson::ct_error::assert_all{} ).then([FNAME] { - SUBINFO(test, "finish"); - }); + SUBINFO(test, "finish"); + }); }); } @@ -261,13 +263,14 @@ protected: TMTestState(std::size_t num_main_devices, std::size_t num_cold_devices) : EphemeralTestState(num_main_devices, num_cold_devices) {} - virtual void _init() override { + virtual seastar::future<> _init() override { auto sec_devices = devices->get_secondary_devices(); auto p_dev = devices->get_primary_device(); tm = make_transaction_manager(p_dev, sec_devices, true); epm = tm->get_epm(); lba_manager = tm->get_lba_manager(); cache = tm->get_cache(); + return seastar::now(); } virtual void _destroy() override { @@ -407,10 +410,10 @@ protected: SeaStoreTestState() : EphemeralTestState(1, 0) {} - virtual void _init() final { + virtual seastar::future<> _init() final { seastore = make_test_seastore( - devices->get_primary_device_ref(), std::make_unique(mdstore_state.get_mdstore())); + return seastore->test_start(devices->get_primary_device_ref()); } virtual void _destroy() final { -- 2.39.5