From: Chunmei Liu <chunmei.liu@ibm.com>
Date: Wed, 3 Sep 2025 04:28:43 +0000 (+0000)
Subject: crimson/common/options: add seastore_require_partition_count_match_reactor_count... 
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=635dfcb35f10101783302cf86e885205f83fef88;p=ceph-ci.git

crimson/common/options: add seastore_require_partition_count_match_reactor_count in crimson.yaml.in

Signed-off-by: Chunmei Liu <chunmei.liu@ibm.com>
---

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index 031113ae7f5..0e11ba6a4aa 100644
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -258,3 +258,8 @@ options:
         internal checksum feature without using sever CPU then enable if available,
         set to true to disable unconditionally.
   default: true
+- name: seastore_require_partition_count_match_reactor_count
+  type: bool
+  level: advanced
+  desc: enable osd shards changes upon restart.
+  default: true
diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h
index eb932fca72a..19af576d700 100644
--- a/src/crimson/os/futurized_store.h
+++ b/src/crimson/os/futurized_store.h
@@ -13,6 +13,7 @@
 #include <seastar/core/sharded.hh>
 
 #include "os/Transaction.h"
+#include "crimson/common/config_proxy.h"
 #include "crimson/common/local_shared_foreign_ptr.h"
 #include "crimson/common/smp_helpers.h"
 #include "crimson/common/smp_helpers.h"
@@ -269,6 +270,9 @@ auto with_store(crimson::os::FuturizedStore::StoreShardRef store, Args&&... args
   constexpr bool is_seastar_future = seastar::is_future<raw_return_type>::value && !is_errorator;
   constexpr bool is_plain = !is_errorator && !is_seastar_future;
   const auto original_core = seastar::this_shard_id();
+  if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+    ceph_assert(store.get_owner_shard() == seastar::this_shard_id());
+  }
   if (store.get_owner_shard() == seastar::this_shard_id()) {
     if constexpr (is_plain) {
       return seastar::make_ready_future<raw_return_type>(
diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 6d502afd3f4..18fb3410b65 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -235,6 +235,13 @@ seastar::future<> SeaStore::get_shard_nums()
       });
     INFO("seastore shard nums {}", shard_nums);
     store_shard_nums = shard_nums;
+    if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+      INFO("seastore doesn't allow shard change");
+      if (store_shard_nums != seastar::smp::count) {
+        INFO("seastore shards {} do not match seastar::smp {}", store_shard_nums, seastar::smp::count);
+        ceph_abort_msg("seastore_require_partition_count_match_reactor_count is true, seastore shards do not match seastar::smp");
+      }
+    }
     co_return;
   }
 }
diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h
index ba28c836777..8fa39f8d2ee 100644
--- a/src/crimson/os/seastore/segment_manager.h
+++ b/src/crimson/os/seastore/segment_manager.h
@@ -57,6 +57,9 @@ struct block_sm_superblock_t {
   }
 
   void validate() const {
+    if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+      ceph_assert(shard_num == seastar::smp::count);
+    }
     ceph_assert(block_size > 0);
     ceph_assert(segment_size > 0 &&
                 segment_size % block_size == 0);
diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index 01504d2cd65..55e75e77db0 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -127,38 +127,42 @@ seastar::future<std::pair<core_id_t, unsigned int>> PGShardMapping::get_or_creat
         ceph_assert_always(primary_mapping.core_to_num_pgs.end() != count_iter);
         ++(count_iter->second);
 
-        if (seastar::smp::count > store_shard_nums ) {
-          auto alien_iter = primary_mapping.core_alien_to_num_pgs.find(core_to_update);
-          auto core_iter = std::min_element(
-            alien_iter->second.begin(),
-            alien_iter->second.end(),
-            [](const auto &left, const auto &right) {
-              return left.second < right.second;
-            }
-          );
-          core_iter->second++;
-          core_to_update = core_iter->first;
-        }
-        if (seastar::smp::count >= store_shard_nums) {
-          shard_index_update = 0; // use the first store shard index on this core
+        if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+          shard_index_update = 0;
         } else {
-          core_shard_iter = primary_mapping.core_shard_to_num_pgs.find(core_to_update);
-          ceph_assert_always(core_shard_iter != primary_mapping.core_shard_to_num_pgs.end());
-          if (shard_index_update == NULL_STORE_INDEX) {
-            // find the store shard index with the least number of pgs
-            // on this core
-            shard_iter = std::min_element(
-              core_shard_iter->second.begin(),
-              core_shard_iter->second.end(),
+          if (seastar::smp::count > store_shard_nums ) {
+            auto alien_iter = primary_mapping.core_alien_to_num_pgs.find(core_to_update);
+            auto core_iter = std::min_element(
+              alien_iter->second.begin(),
+              alien_iter->second.end(),
               [](const auto &left, const auto &right) {
                 return left.second < right.second;
               }
             );
-            shard_index_update = shard_iter->first; //find the store shard index on this core
+            core_iter->second++;
+            core_to_update = core_iter->first;
+          }
+          if (seastar::smp::count >= store_shard_nums) {
+            shard_index_update = 0; // use the first store shard index on this core
           } else {
-            shard_iter = core_shard_iter->second.find(shard_index_update);
+            core_shard_iter = primary_mapping.core_shard_to_num_pgs.find(core_to_update);
+            ceph_assert_always(core_shard_iter != primary_mapping.core_shard_to_num_pgs.end());
+            if (shard_index_update == NULL_STORE_INDEX) {
+              // find the store shard index with the least number of pgs
+              // on this core
+              shard_iter = std::min_element(
+                core_shard_iter->second.begin(),
+                core_shard_iter->second.end(),
+                [](const auto &left, const auto &right) {
+                  return left.second < right.second;
+                }
+              );
+              shard_index_update = shard_iter->first; //find the store shard index on this core
+            }  else {
+              shard_iter = core_shard_iter->second.find(shard_index_update);
+            }
+            ++(shard_iter->second);
           }
-          ++(shard_iter->second);
         }
         [[maybe_unused]] auto [insert_iter, inserted] =
           primary_mapping.pg_to_core.emplace(pgid, std::make_pair(core_to_update, shard_index_update));
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index ea93900e17f..d7fdb172696 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -489,6 +489,11 @@ public:
   void set_container(seastar::sharded<ShardServices>& ss) { s_container = &ss; }
 
   seastar::future<> get_remote_store() {
+    if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+      ceph_assert(store_shard_nums == seastar::smp::count);
+      ceph_assert(local_state.stores.size() == 1);
+      return seastar::now();
+    }
     if (local_state.stores.empty()) {
       return s_container->invoke_on(
         seastar::this_shard_id() % store_shard_nums,