From 5544b2ebd8b0ec3836511e32c691096527886c3c Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Wed, 31 Aug 2022 18:34:30 +0000 Subject: [PATCH] crimson/osd: make mkfs static Can't access pg_shard_manager state here without invoking start(). Instead, let's just be explicit about meta_coll etc. Signed-off-by: Samuel Just --- src/crimson/osd/main.cc | 2 + src/crimson/osd/osd.cc | 126 ++++++++++++++++++++++------------------ src/crimson/osd/osd.h | 22 +++++-- 3 files changed, 88 insertions(+), 62 deletions(-) diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc index f572cd9fbf5..df3549b3126 100644 --- a/src/crimson/osd/main.cc +++ b/src/crimson/osd/main.cc @@ -325,6 +325,8 @@ int main(int argc, const char* argv[]) osd_uuid.generate_random(); } osd.mkfs( + *store, + whoami, osd_uuid, local_conf().get_val("fsid"), config["osdspec-affinity"].as()).get(); diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 7feaa410c2e..07e15168a56 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -162,27 +162,29 @@ seastar::future<> OSD::open_meta_coll() }); } -seastar::future<> OSD::open_or_create_meta_coll() +seastar::future OSD::open_or_create_meta_coll(FuturizedStore &store) { - return store.open_collection(coll_t::meta()).then([this] (auto ch) { + return store.open_collection(coll_t::meta()).then([&store](auto ch) { if (!ch) { - return store.create_new_collection(coll_t::meta()).then([this] (auto ch) { - pg_shard_manager.init_meta_coll(ch, store); - return seastar::now(); + return store.create_new_collection( + coll_t::meta() + ).then([&store](auto ch) { + return OSDMeta(ch, store); }); } else { - pg_shard_manager.init_meta_coll(ch, store); - return seastar::now(); + return seastar::make_ready_future(ch, store); } }); } seastar::future<> OSD::mkfs( + FuturizedStore &store, + unsigned whoami, uuid_d osd_uuid, uuid_d cluster_fsid, std::string osdspec_affinity) { - return store.start().then([this, osd_uuid] { + return store.start().then([&store, osd_uuid] { return store.mkfs(osd_uuid).handle_error( crimson::stateful_ec::handle([] (const auto& ec) { logger().error("error creating empty object store in {}: ({}) {}", @@ -190,74 +192,86 @@ seastar::future<> OSD::mkfs( ec.value(), ec.message()); std::exit(EXIT_FAILURE); })); - }).then([this] { + }).then([&store] { return store.mount().handle_error( - crimson::stateful_ec::handle([] (const auto& ec) { + crimson::stateful_ec::handle([](const auto& ec) { logger().error("error mounting object store in {}: ({}) {}", local_conf().get_val("osd_data"), ec.value(), ec.message()); std::exit(EXIT_FAILURE); })); - }).then([this] { - return open_or_create_meta_coll(); - }).then([cluster_fsid, this] { + }).then([&store] { + return open_or_create_meta_coll(store); + }).then([&store, whoami, cluster_fsid](auto meta_coll) { + OSDSuperblock superblock; superblock.cluster_fsid = cluster_fsid; superblock.osd_fsid = store.get_fsid(); superblock.whoami = whoami; superblock.compat_features = get_osd_initial_compat_set(); - return _write_superblock(); - }).then([cluster_fsid, this] { + return _write_superblock( + store, std::move(meta_coll), std::move(superblock)); + }).then([&store, cluster_fsid] { return store.write_meta("ceph_fsid", cluster_fsid.to_string()); - }).then([this] { + }).then([&store] { return store.write_meta("magic", CEPH_OSD_ONDISK_MAGIC); - }).then([this] { + }).then([&store, whoami] { return store.write_meta("whoami", std::to_string(whoami)); - }).then([this] { - return _write_key_meta(); - }).then([this, osdspec_affinity=std::move(osdspec_affinity)] { + }).then([&store] { + return _write_key_meta(store); + }).then([&store, osdspec_affinity=std::move(osdspec_affinity)] { return store.write_meta("osdspec_affinity", osdspec_affinity); - }).then([this] { + }).then([&store] { return store.write_meta("ready", "ready"); - }).then([cluster_fsid, this] { + }).then([&store, whoami, cluster_fsid] { fmt::print("created object store {} for osd.{} fsid {}\n", local_conf().get_val("osd_data"), whoami, cluster_fsid); - return seastar::now(); + return store.umount(); + }).then([&store] { + return store.stop(); }); } -seastar::future<> OSD::_write_superblock() +seastar::future<> OSD::_write_superblock( + FuturizedStore &store, + OSDMeta meta_coll, + OSDSuperblock superblock) { - return pg_shard_manager.get_meta_coll().load_superblock( - ).safe_then([this](OSDSuperblock&& sb) { - if (sb.cluster_fsid != superblock.cluster_fsid) { - logger().error("provided cluster fsid {} != superblock's {}", - sb.cluster_fsid, superblock.cluster_fsid); - throw std::invalid_argument("mismatched fsid"); - } - if (sb.whoami != superblock.whoami) { - logger().error("provided osd id {} != superblock's {}", - sb.whoami, superblock.whoami); - throw std::invalid_argument("mismatched osd id"); - } - }).handle_error( - crimson::ct_error::enoent::handle([this] { - // meta collection does not yet, create superblock - logger().info( - "{} writing superblock cluster_fsid {} osd_fsid {}", - "_write_superblock", - superblock.cluster_fsid, - superblock.osd_fsid); - ceph::os::Transaction t; - pg_shard_manager.get_meta_coll().create(t); - pg_shard_manager.get_meta_coll().store_superblock(t, superblock); - logger().debug("OSD::_write_superblock: do_transaction..."); - return store.do_transaction( - pg_shard_manager.get_meta_coll().collection(), - std::move(t)); - }), - crimson::ct_error::assert_all("_write_superbock error") - ); + return seastar::do_with( + std::move(meta_coll), + std::move(superblock), + [&store](auto &meta_coll, auto &superblock) { + return meta_coll.load_superblock( + ).safe_then([&superblock](OSDSuperblock&& sb) { + if (sb.cluster_fsid != superblock.cluster_fsid) { + logger().error("provided cluster fsid {} != superblock's {}", + sb.cluster_fsid, superblock.cluster_fsid); + throw std::invalid_argument("mismatched fsid"); + } + if (sb.whoami != superblock.whoami) { + logger().error("provided osd id {} != superblock's {}", + sb.whoami, superblock.whoami); + throw std::invalid_argument("mismatched osd id"); + } + }).handle_error( + crimson::ct_error::enoent::handle([&store, &meta_coll, &superblock] { + // meta collection does not yet, create superblock + logger().info( + "{} writing superblock cluster_fsid {} osd_fsid {}", + "_write_superblock", + superblock.cluster_fsid, + superblock.osd_fsid); + ceph::os::Transaction t; + meta_coll.create(t); + meta_coll.store_superblock(t, superblock); + logger().debug("OSD::_write_superblock: do_transaction..."); + return store.do_transaction( + meta_coll.collection(), + std::move(t)); + }), + crimson::ct_error::assert_all("_write_superbock error") + ); + }); } // this `to_string` sits in the `crimson::osd` namespace, so we don't brake @@ -267,14 +281,14 @@ static std::string to_string(const seastar::temporary_buffer& temp_buf) return {temp_buf.get(), temp_buf.size()}; } -seastar::future<> OSD::_write_key_meta() +seastar::future<> OSD::_write_key_meta(FuturizedStore &store) { if (auto key = local_conf().get_val("key"); !std::empty(key)) { return store.write_meta("osd_key", key); } else if (auto keyfile = local_conf().get_val("keyfile"); !std::empty(keyfile)) { - return read_file(keyfile).then([this] (const auto& temp_buf) { + return read_file(keyfile).then([&store](const auto& temp_buf) { // it's on a truly cold path, so don't worry about memcpy. return store.write_meta("osd_key", to_string(temp_buf)); }).handle_exception([keyfile] (auto ep) { diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index 18ddd387b2f..0db21eb84bb 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -120,10 +120,15 @@ public: ~OSD() final; seastar::future<> open_meta_coll(); - seastar::future<> open_or_create_meta_coll(); - seastar::future<> mkfs(uuid_d osd_uuid, - uuid_d cluster_fsid, - std::string osdspec_affinity); + static seastar::future open_or_create_meta_coll( + crimson::os::FuturizedStore &store + ); + static seastar::future<> mkfs( + crimson::os::FuturizedStore &store, + unsigned whoami, + uuid_d osd_uuid, + uuid_d cluster_fsid, + std::string osdspec_affinity); seastar::future<> start(); seastar::future<> stop(); @@ -139,8 +144,13 @@ public: uint64_t send_pg_stats(); private: - seastar::future<> _write_superblock(); - seastar::future<> _write_key_meta(); + static seastar::future<> _write_superblock( + crimson::os::FuturizedStore &store, + OSDMeta meta, + OSDSuperblock superblock); + static seastar::future<> _write_key_meta( + crimson::os::FuturizedStore &store + ); seastar::future<> start_boot(); seastar::future<> _preboot(version_t oldest_osdmap, version_t newest_osdmap); seastar::future<> _send_boot(); -- 2.39.5