From 3b48669b8e01db705a678e071c77e8a2e1bb7b77 Mon Sep 17 00:00:00 2001 From: Chunmei Liu Date: Wed, 3 Sep 2025 04:28:43 +0000 Subject: [PATCH] crimson/common/options: add seastore_enable_shard_change in crimson.yaml.in Signed-off-by: Chunmei Liu --- src/common/options/crimson.yaml.in | 5 +++ src/crimson/os/futurized_store.h | 4 ++ src/crimson/os/seastore/seastore.cc | 7 +++ src/crimson/os/seastore/segment_manager.h | 3 ++ src/crimson/osd/pg_map.cc | 54 ++++++++++++----------- src/crimson/osd/shard_services.h | 5 +++ 6 files changed, 53 insertions(+), 25 deletions(-) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index 031113ae7f5..61810c56b16 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -258,3 +258,8 @@ options: internal checksum feature without using sever CPU then enable if available, set to true to disable unconditionally. default: true +- name: seastore_enable_shard_change + type: bool + level: advanced + desc: enable osd shards changes upon restart. + default: true diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h index 6261ace138e..898d66c2550 100644 --- a/src/crimson/os/futurized_store.h +++ b/src/crimson/os/futurized_store.h @@ -13,6 +13,7 @@ #include #include "os/Transaction.h" +#include "crimson/common/config_proxy.h" #include "crimson/common/local_shared_foreign_ptr.h" #include "crimson/common/smp_helpers.h" #include "crimson/common/smp_helpers.h" @@ -261,6 +262,9 @@ auto with_store(crimson::os::FuturizedStore::StoreShardRef store, Args&&... args constexpr bool is_seastar_future = seastar::is_future::value && !is_errorator; constexpr bool is_plain = !is_errorator && !is_seastar_future; const auto original_core = seastar::this_shard_id(); + if(!crimson::common::get_conf("seastore_enable_shard_change")) { + ceph_assert(store.get_owner_shard() == seastar::this_shard_id()); + } if (store.get_owner_shard() == seastar::this_shard_id()) { if constexpr (is_plain) { return seastar::make_ready_future( diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index b8329b077c7..c191a1dca94 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -233,6 +233,13 @@ seastar::future<> SeaStore::get_shard_nums() }); INFO("seastore shard nums {}", shard_nums); store_shard_nums = shard_nums; + if(!crimson::common::get_conf("seastore_enable_shard_change")) { + INFO("seastore_enable_shard_change is false"); + if (store_shard_nums != seastar::smp::count) { + INFO("seastore shards {} do not match seastar::smp {}", store_shard_nums, seastar::smp::count); + ceph_abort_msg("seastore_enable_shard_change is false, seastore shards do not match seastar::smp"); + } + } co_return; } } diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h index ba28c836777..41fb2dedddd 100644 --- a/src/crimson/os/seastore/segment_manager.h +++ b/src/crimson/os/seastore/segment_manager.h @@ -57,6 +57,9 @@ struct block_sm_superblock_t { } void validate() const { + if(!crimson::common::get_conf("seastore_enable_shard_change")) { + ceph_assert(shard_num == seastar::smp::count); + } ceph_assert(block_size > 0); ceph_assert(segment_size > 0 && segment_size % block_size == 0); diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc index 01504d2cd65..c7e7de18e87 100644 --- a/src/crimson/osd/pg_map.cc +++ b/src/crimson/osd/pg_map.cc @@ -127,38 +127,42 @@ seastar::future> PGShardMapping::get_or_creat ceph_assert_always(primary_mapping.core_to_num_pgs.end() != count_iter); ++(count_iter->second); - if (seastar::smp::count > store_shard_nums ) { - auto alien_iter = primary_mapping.core_alien_to_num_pgs.find(core_to_update); - auto core_iter = std::min_element( - alien_iter->second.begin(), - alien_iter->second.end(), - [](const auto &left, const auto &right) { - return left.second < right.second; - } - ); - core_iter->second++; - core_to_update = core_iter->first; - } - if (seastar::smp::count >= store_shard_nums) { - shard_index_update = 0; // use the first store shard index on this core + if(!crimson::common::get_conf("seastore_enable_shard_change")) { + shard_index_update = 0; } else { - core_shard_iter = primary_mapping.core_shard_to_num_pgs.find(core_to_update); - ceph_assert_always(core_shard_iter != primary_mapping.core_shard_to_num_pgs.end()); - if (shard_index_update == NULL_STORE_INDEX) { - // find the store shard index with the least number of pgs - // on this core - shard_iter = std::min_element( - core_shard_iter->second.begin(), - core_shard_iter->second.end(), + if (seastar::smp::count > store_shard_nums ) { + auto alien_iter = primary_mapping.core_alien_to_num_pgs.find(core_to_update); + auto core_iter = std::min_element( + alien_iter->second.begin(), + alien_iter->second.end(), [](const auto &left, const auto &right) { return left.second < right.second; } ); - shard_index_update = shard_iter->first; //find the store shard index on this core + core_iter->second++; + core_to_update = core_iter->first; + } + if (seastar::smp::count >= store_shard_nums) { + shard_index_update = 0; // use the first store shard index on this core } else { - shard_iter = core_shard_iter->second.find(shard_index_update); + core_shard_iter = primary_mapping.core_shard_to_num_pgs.find(core_to_update); + ceph_assert_always(core_shard_iter != primary_mapping.core_shard_to_num_pgs.end()); + if (shard_index_update == NULL_STORE_INDEX) { + // find the store shard index with the least number of pgs + // on this core + shard_iter = std::min_element( + core_shard_iter->second.begin(), + core_shard_iter->second.end(), + [](const auto &left, const auto &right) { + return left.second < right.second; + } + ); + shard_index_update = shard_iter->first; //find the store shard index on this core + } else { + shard_iter = core_shard_iter->second.find(shard_index_update); + } + ++(shard_iter->second); } - ++(shard_iter->second); } [[maybe_unused]] auto [insert_iter, inserted] = primary_mapping.pg_to_core.emplace(pgid, std::make_pair(core_to_update, shard_index_update)); diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h index 04d2a006b30..df7820c5530 100644 --- a/src/crimson/osd/shard_services.h +++ b/src/crimson/osd/shard_services.h @@ -477,6 +477,11 @@ public: void set_container(seastar::sharded& ss) { s_container = &ss; } seastar::future<> get_remote_store() { + if(!crimson::common::get_conf("seastore_enable_shard_change")) { + ceph_assert(store_shard_nums == seastar::smp::count); + ceph_assert(local_state.stores.size() == 1); + return seastar::now(); + } if (local_state.stores.empty()) { return s_container->invoke_on( seastar::this_shard_id() % store_shard_nums, -- 2.39.5