crimson/common/options: add seastore_require_partition_count_match_reactor_count...

Signed-off-by: Chunmei Liu <chunmei.liu@ibm.com>

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index 031113ae7f55965f0b28e888b27c66aa583f6548..0e11ba6a4aa8469620387a88d19981da801a9db3 100644 (file)
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -258,3 +258,8 @@ options:
         internal checksum feature without using sever CPU then enable if available,
         set to true to disable unconditionally.
   default: true
+- name: seastore_require_partition_count_match_reactor_count
+  type: bool
+  level: advanced
+  desc: enable osd shards changes upon restart.
+  default: true

diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h
index eb932fca72a33b6095169e8fdeb9d6cfd1db19e5..19af576d700187b409c1bca82b45d6c6a77b0e76 100644 (file)
--- a/src/crimson/os/futurized_store.h
+++ b/src/crimson/os/futurized_store.h
@@ -13,6 +13,7 @@
 #include <seastar/core/sharded.hh>
 
 #include "os/Transaction.h"
+#include "crimson/common/config_proxy.h"
 #include "crimson/common/local_shared_foreign_ptr.h"
 #include "crimson/common/smp_helpers.h"
 #include "crimson/common/smp_helpers.h"
@@ -269,6 +270,9 @@ auto with_store(crimson::os::FuturizedStore::StoreShardRef store, Args&&... args
   constexpr bool is_seastar_future = seastar::is_future<raw_return_type>::value && !is_errorator;
   constexpr bool is_plain = !is_errorator && !is_seastar_future;
   const auto original_core = seastar::this_shard_id();
+  if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+    ceph_assert(store.get_owner_shard() == seastar::this_shard_id());
+  }
   if (store.get_owner_shard() == seastar::this_shard_id()) {
     if constexpr (is_plain) {
       return seastar::make_ready_future<raw_return_type>(

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 6d502afd3f4e466c654fdc687d7dd19080fdb1e0..18fb3410b65979cfa39b74bd07de6bacc3abcd4c 100644 (file)
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -235,6 +235,13 @@ seastar::future<> SeaStore::get_shard_nums()
       });
     INFO("seastore shard nums {}", shard_nums);
     store_shard_nums = shard_nums;
+    if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+      INFO("seastore doesn't allow shard change");
+      if (store_shard_nums != seastar::smp::count) {
+        INFO("seastore shards {} do not match seastar::smp {}", store_shard_nums, seastar::smp::count);
+        ceph_abort_msg("seastore_require_partition_count_match_reactor_count is true, seastore shards do not match seastar::smp");
+      }
+    }
     co_return;
   }
 }

diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h
index ba28c83677729c3b417f90323c92cf3218569938..8fa39f8d2eebb2fa6ab8bf4eed4fba058fa34767 100644 (file)
--- a/src/crimson/os/seastore/segment_manager.h
+++ b/src/crimson/os/seastore/segment_manager.h
@@ -57,6 +57,9 @@ struct block_sm_superblock_t {
   }
 
   void validate() const {
+    if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+      ceph_assert(shard_num == seastar::smp::count);
+    }
     ceph_assert(block_size > 0);
     ceph_assert(segment_size > 0 &&
                 segment_size % block_size == 0);

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index 01504d2cd6512c64a8d62bfcc672930a14308770..55e75e77db002f7335c9a0d34a31b9be8977dd35 100644 (file)
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -127,38 +127,42 @@ seastar::future<std::pair<core_id_t, unsigned int>> PGShardMapping::get_or_creat
         ceph_assert_always(primary_mapping.core_to_num_pgs.end() != count_iter);
         ++(count_iter->second);
 
-        if (seastar::smp::count > store_shard_nums ) {
-          auto alien_iter = primary_mapping.core_alien_to_num_pgs.find(core_to_update);
-          auto core_iter = std::min_element(
-            alien_iter->second.begin(),
-            alien_iter->second.end(),
-            [](const auto &left, const auto &right) {
-              return left.second < right.second;
-            }
-          );
-          core_iter->second++;
-          core_to_update = core_iter->first;
-        }
-        if (seastar::smp::count >= store_shard_nums) {
-          shard_index_update = 0; // use the first store shard index on this core
+        if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+          shard_index_update = 0;
         } else {
-          core_shard_iter = primary_mapping.core_shard_to_num_pgs.find(core_to_update);
-          ceph_assert_always(core_shard_iter != primary_mapping.core_shard_to_num_pgs.end());
-          if (shard_index_update == NULL_STORE_INDEX) {
-            // find the store shard index with the least number of pgs
-            // on this core
-            shard_iter = std::min_element(
-              core_shard_iter->second.begin(),
-              core_shard_iter->second.end(),
+          if (seastar::smp::count > store_shard_nums ) {
+            auto alien_iter = primary_mapping.core_alien_to_num_pgs.find(core_to_update);
+            auto core_iter = std::min_element(
+              alien_iter->second.begin(),
+              alien_iter->second.end(),
               [](const auto &left, const auto &right) {
                 return left.second < right.second;
               }
             );
-            shard_index_update = shard_iter->first; //find the store shard index on this core
+            core_iter->second++;
+            core_to_update = core_iter->first;
+          }
+          if (seastar::smp::count >= store_shard_nums) {
+            shard_index_update = 0; // use the first store shard index on this core
           } else {
-            shard_iter = core_shard_iter->second.find(shard_index_update);
+            core_shard_iter = primary_mapping.core_shard_to_num_pgs.find(core_to_update);
+            ceph_assert_always(core_shard_iter != primary_mapping.core_shard_to_num_pgs.end());
+            if (shard_index_update == NULL_STORE_INDEX) {
+              // find the store shard index with the least number of pgs
+              // on this core
+              shard_iter = std::min_element(
+                core_shard_iter->second.begin(),
+                core_shard_iter->second.end(),
+                [](const auto &left, const auto &right) {
+                  return left.second < right.second;
+                }
+              );
+              shard_index_update = shard_iter->first; //find the store shard index on this core
+            }  else {
+              shard_iter = core_shard_iter->second.find(shard_index_update);
+            }
+            ++(shard_iter->second);
           }
-          ++(shard_iter->second);
         }
         [[maybe_unused]] auto [insert_iter, inserted] =
           primary_mapping.pg_to_core.emplace(pgid, std::make_pair(core_to_update, shard_index_update));

diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index ea93900e17fbed2aa3e919121c7558bb3a7bc96b..d7fdb172696092dce14e5da5fad590fa8e0b4171 100644 (file)
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -489,6 +489,11 @@ public:
   void set_container(seastar::sharded<ShardServices>& ss) { s_container = &ss; }
 
   seastar::future<> get_remote_store() {
+    if(crimson::common::get_conf<bool>("seastore_require_partition_count_match_reactor_count")) {
+      ceph_assert(store_shard_nums == seastar::smp::count);
+      ceph_assert(local_state.stores.size() == 1);
+      return seastar::now();
+    }
     if (local_state.stores.empty()) {
       return s_container->invoke_on(
         seastar::this_shard_id() % store_shard_nums,