From: chunmei-liu Date: Wed, 11 Jan 2023 05:39:16 +0000 (-0800) Subject: crimson/os: make load_pgs() load pg on proper core X-Git-Tag: v18.1.0~536^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=bc5876e3b975f01525243616e0d04fe46605aa58;p=ceph-ci.git crimson/os: make load_pgs() load pg on proper core Signed-off-by: chunmei-liu --- diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc index dd5b5408411..a458d321b86 100644 --- a/src/crimson/os/alienstore/alien_store.cc +++ b/src/crimson/os/alienstore/alien_store.cc @@ -262,7 +262,7 @@ seastar::future AlienStore::open_collection(const coll_t& cid) }); } -seastar::future> AlienStore::list_collections() +seastar::future> AlienStore::list_collections() { logger().debug("{}", __func__); assert(tp); @@ -270,9 +270,14 @@ seastar::future> AlienStore::list_collections() return do_with_op_gate(std::vector{}, [this] (auto &ls) { return tp->submit([this, &ls] { return store->list_collections(ls); - }).then([&ls] (int r) { + }).then([&ls] (int r) -> seastar::future> { assert(r == 0); - return seastar::make_ready_future>(std::move(ls)); + std::vector ret; + ret.resize(ls.size()); + std::transform( + ls.begin(), ls.end(), ret.begin(), + [](auto p) { return std::make_pair(p, NULL_CORE); }); + return seastar::make_ready_future>(std::move(ret)); }); }); } diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h index 1b6a4276479..f299ffda811 100644 --- a/src/crimson/os/alienstore/alien_store.h +++ b/src/crimson/os/alienstore/alien_store.h @@ -19,6 +19,7 @@ class Transaction; } namespace crimson::os { +using coll_core_t = FuturizedStore::coll_core_t; class AlienStore final : public FuturizedStore { public: AlienStore(const std::string& type, @@ -69,7 +70,7 @@ public: seastar::future create_new_collection(const coll_t& cid) final; seastar::future open_collection(const coll_t& cid) final; - seastar::future> list_collections() final; + seastar::future> list_collections() final; seastar::future<> do_transaction_no_callbacks( CollectionRef c, diff --git a/src/crimson/os/cyanstore/cyan_store.cc b/src/crimson/os/cyanstore/cyan_store.cc index 5336f1eb971..daeb1e56767 100644 --- a/src/crimson/os/cyanstore/cyan_store.cc +++ b/src/crimson/os/cyanstore/cyan_store.cc @@ -123,17 +123,17 @@ seastar::future<> CyanStore::ShardStores::mkfs() return crimson::write_file(std::move(bl), fn); } -seastar::future> +seastar::future> CyanStore::list_collections() { - return seastar::do_with(std::vector{}, [this](auto &collections) { + return seastar::do_with(std::vector{}, [this](auto &collections) { return shard_stores.map([](auto &local_store) { return local_store.list_collections(); - }).then([&collections](std::vector> results) { + }).then([&collections](std::vector> results) { for (auto& colls : results) { collections.insert(collections.end(), colls.begin(), colls.end()); } - return seastar::make_ready_future>( + return seastar::make_ready_future>( std::move(collections)); }); }); @@ -233,14 +233,14 @@ CyanStore::ShardStores::open_collection(const coll_t& cid) return seastar::make_ready_future(_get_collection(cid)); } -seastar::future> +seastar::future> CyanStore::ShardStores::list_collections() { - std::vector collections; + std::vector collections; for (auto& coll : coll_map) { - collections.push_back(coll.first); + collections.push_back(std::make_pair(coll.first, seastar::this_shard_id())); } - return seastar::make_ready_future>(std::move(collections)); + return seastar::make_ready_future>(std::move(collections)); } CyanStore::read_errorator::future diff --git a/src/crimson/os/cyanstore/cyan_store.h b/src/crimson/os/cyanstore/cyan_store.h index 738157e8a85..715f8ae8caf 100644 --- a/src/crimson/os/cyanstore/cyan_store.h +++ b/src/crimson/os/cyanstore/cyan_store.h @@ -25,7 +25,7 @@ class Transaction; } namespace crimson::os { - +using coll_core_t = FuturizedStore::coll_core_t; class CyanStore final : public FuturizedStore { class ShardStores { public: @@ -92,7 +92,7 @@ class CyanStore final : public FuturizedStore { seastar::future open_collection(const coll_t& cid); - seastar::future> list_collections(); + seastar::future> list_collections(); seastar::future<> do_transaction_no_callbacks( CollectionRef ch, @@ -195,7 +195,7 @@ public: }); } - seastar::future> list_collections() final; + seastar::future> list_collections() final; // public interfaces called by each shard osd public: diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h index acde6447ed6..2fae5520275 100644 --- a/src/crimson/os/futurized_store.h +++ b/src/crimson/os/futurized_store.h @@ -12,6 +12,7 @@ #include "os/Transaction.h" #include "crimson/common/smp_helpers.h" +#include "crimson/common/smp_helpers.h" #include "crimson/osd/exceptions.h" #include "include/buffer_fwd.h" #include "include/uuid.h" @@ -106,7 +107,8 @@ public: virtual seastar::future create_new_collection(const coll_t& cid) = 0; virtual seastar::future open_collection(const coll_t& cid) = 0; - virtual seastar::future> list_collections() = 0; + using coll_core_t = std::pair; + virtual seastar::future> list_collections() = 0; protected: virtual seastar::future<> do_transaction_no_callbacks( @@ -298,7 +300,7 @@ public: seastar::future open_collection(const coll_t &cid) final { return proxy(&T::open_collection, cid); } - seastar::future> list_collections() final { + seastar::future> list_collections() final { return proxy(&T::list_collections); } seastar::future<> do_transaction_no_callbacks( diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 27c1cc51549..86917c0ebc6 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -574,9 +574,10 @@ seastar::future SeaStore::open_collection(const coll_t& cid) { LOG_PREFIX(SeaStore::open_collection); DEBUG("{}", cid); - return list_collections().then([cid, this] (auto colls) { - if (auto found = std::find(colls.begin(), colls.end(), cid); - found != colls.end()) { + return list_collections().then([cid, this] (auto colls_cores) { + if (auto found = std::find(colls_cores.begin(), colls_cores.end(), + std::make_pair(cid, NULL_CORE)); + found != colls_cores.end()) { return seastar::make_ready_future(_get_collection(cid)); } else { return seastar::make_ready_future(); @@ -584,10 +585,10 @@ seastar::future SeaStore::open_collection(const coll_t& cid) }); } -seastar::future> SeaStore::list_collections() +seastar::future> SeaStore::list_collections() { return seastar::do_with( - std::vector(), + std::vector(), [this](auto &ret) { return repeat_eagain([this, &ret] { return transaction_manager->with_transaction_intr( @@ -602,11 +603,11 @@ seastar::future> SeaStore::list_collections() ret.resize(colls.size()); std::transform( colls.begin(), colls.end(), ret.begin(), - [](auto p) { return p.first; }); + [](auto p) { return std::make_pair(p.first, NULL_CORE); }); }); }); }).safe_then([&ret] { - return seastar::make_ready_future>(ret); + return seastar::make_ready_future>(ret); }); } ).handle_error( diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index 9b8833e73a5..36b53683450 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -55,6 +55,7 @@ struct col_obj_ranges_t { ghobject_t obj_end; }; +using coll_core_t = FuturizedStore::coll_core_t; class SeaStore final : public FuturizedStore { public: class MDStore { @@ -145,7 +146,7 @@ public: seastar::future create_new_collection(const coll_t& cid) final; seastar::future open_collection(const coll_t& cid) final; - seastar::future> list_collections() final; + seastar::future> list_collections() final; seastar::future<> do_transaction_no_callbacks( CollectionRef ch, diff --git a/src/crimson/osd/pg_map.h b/src/crimson/osd/pg_map.h index ed753749a55..3af746e6052 100644 --- a/src/crimson/osd/pg_map.h +++ b/src/crimson/osd/pg_map.h @@ -33,19 +33,27 @@ public: } /// Returns mapping for pgid, creates new one if it doesn't already exist - core_id_t maybe_create_pg(spg_t pgid) { - auto [insert_iter, inserted] = pg_to_core.emplace(pgid, NULL_CORE); + core_id_t maybe_create_pg(spg_t pgid, core_id_t core = NULL_CORE) { + auto [insert_iter, inserted] = pg_to_core.emplace(pgid, core); if (!inserted) { ceph_assert_always(insert_iter->second != NULL_CORE); + if (core != NULL_CORE) { + ceph_assert_always(insert_iter->second == core); + } return insert_iter->second; } else { ceph_assert_always(core_to_num_pgs.size() > 0); - auto core_iter = std::min_element( - core_to_num_pgs.begin(), - core_to_num_pgs.end(), - [](const auto &left, const auto &right) { - return left.second < right.second; - }); + std::map::iterator core_iter; + if (core == NULL_CORE) { + core_iter = std::min_element( + core_to_num_pgs.begin(), + core_to_num_pgs.end(), + [](const auto &left, const auto &right) { + return left.second < right.second; + }); + } else { + core_iter = core_to_num_pgs.find(core); + } ceph_assert_always(core_to_num_pgs.end() != core_iter); insert_iter->second = core_iter->first; core_iter->second++; diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc index afe9d0a5014..c81ea8a01f4 100644 --- a/src/crimson/osd/pg_shard_manager.cc +++ b/src/crimson/osd/pg_shard_manager.cc @@ -49,15 +49,16 @@ seastar::future<> PGShardManager::load_pgs() { ceph_assert(seastar::this_shard_id() == PRIMARY_CORE); return get_local_state().store.list_collections( - ).then([this](auto colls) { + ).then([this](auto colls_cores) { return seastar::parallel_for_each( - colls, - [this](auto coll) { + colls_cores, + [this](auto coll_core) { + auto[coll, shard_core] = coll_core; spg_t pgid; if (coll.is_pg(&pgid)) { auto core = get_osd_singleton_state( ).pg_to_shard_mapping.maybe_create_pg( - pgid); + pgid, shard_core); return with_remote_shard_state( core, [pgid]( diff --git a/src/test/crimson/seastore/test_seastore.cc b/src/test/crimson/seastore/test_seastore.cc index a2e450de4ae..fc49edc54ab 100644 --- a/src/test/crimson/seastore/test_seastore.cc +++ b/src/test/crimson/seastore/test_seastore.cc @@ -605,10 +605,15 @@ TEST_F(seastore_test_t, collection_create_list_remove) t.create_collection(test_coll, 4); do_transaction(std::move(t)); } - auto collections = seastore->list_collections().get0(); - EXPECT_EQ(collections.size(), 2); - EXPECT_TRUE(contains(collections, coll_name)); - EXPECT_TRUE(contains(collections, test_coll)); + auto colls_cores = seastore->list_collections().get0(); + std::vector colls; + colls.resize(colls_cores.size()); + std::transform( + colls_cores.begin(), colls_cores.end(), colls.begin(), + [](auto p) { return p.first; }); + EXPECT_EQ(colls.size(), 2); + EXPECT_TRUE(contains(colls, coll_name)); + EXPECT_TRUE(contains(colls, test_coll)); } { @@ -617,9 +622,14 @@ TEST_F(seastore_test_t, collection_create_list_remove) t.remove_collection(test_coll); do_transaction(std::move(t)); } - auto collections = seastore->list_collections().get0(); - EXPECT_EQ(collections.size(), 1); - EXPECT_TRUE(contains(collections, coll_name)); + auto colls_cores = seastore->list_collections().get0(); + std::vector colls; + colls.resize(colls_cores.size()); + std::transform( + colls_cores.begin(), colls_cores.end(), colls.begin(), + [](auto p) { return p.first; }); + EXPECT_EQ(colls.size(), 1); + EXPECT_TRUE(contains(colls, coll_name)); } }); }