crimson: introduce crimson_seastar_num_threads

    This new option is there to allow *basic* cluster deployments.
    crimson_seastar_num_threads can be used globally for all the OSDs in the cluster:
    ```
    osd:
      crimson_seastar_num_threads: <n>
      crimson_alien_op_num_threads: <m>
    ```

    As a result, all the available CPUs will be allocated to *both* seastar reactor threads
    and to Alienstore threads - without any exclusion.

    Notes:

    * The core allocation will most likely overlap between OSDS and/or Seastar and Alienstore.
    An optiomal deployment requires `crimson_alien_thread_cpu_cores`
    and `crimson_seastar_cpu_cores` to be set for each OSD based on the host its located at.

    * Seastar reactor number (smp::count) will be deduced directly from crimson_seastar_num_threads

    ---

    Documentation followup: https://github.com/ceph/ceph/pull/57352

---
    ### Option 1 (Optimal)
    * CPUs are pinned to the provided sets (seastar and alien cpu sets)
    * thread-affinity = 1 (seastar's default).
    * smp (smp::count / reactor count): `seastar_cpu_set.size()`
    * Seastar reactor threads: `seastar_cpu_set.size()`
    * Alienstore threads num: `crimson_alien_op_num_threads`
    ---
    ### Option 2 (Basic):
    * No CPU pinning.
    * thread-affinity = 0
    * smp (smp::count / reactor count) = `crimson_seastar_num_threads`
    * Seastar reactor threads num: `crimson_seastar_num_threads`
    * Alienstore threads num: `crimson_alien_op_num_threads`
---
    Fixes: https://tracker.ceph.com/issues/65752
    Signed-off-by: Matan Breizman <mbreizma@redhat.com>
(cherry picked from commit e0b6fff4f3bf9495fc0fdc17e5ea1f53ffa1d423)

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index 7030780d9a9fdc8022a5d0542ac8bee08700dc87..32b6873f0c2330a7e71811b365ce65e096dd4c4a 100644 (file)
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -31,6 +31,15 @@ options:
   desc: CPU cores on which alienstore threads will run in cpuset(7) format
   flags:
   - startup
+- name: crimson_seastar_num_threads
+  type: uint
+  level: advanced
+  default: 0
+  desc: The number of threads for serving seastar reactors without CPU pinning, overridden if crimson_seastar_cpu_cores is set
+  flags:
+  - startup
+  min: 0
+  max: 32
 - name: crimson_osd_stat_interval
   type: int
   level: advanced

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 2139a2dd25856d2bd7d315d0c2a473e2e4483f27..2fac9a89ee4ef729604a0d83b33e12698eb63c5d 100644 (file)
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -101,19 +101,23 @@ seastar::future<> AlienStore::start()
   if (!store) {
     ceph_abort_msgf("unsupported objectstore type: %s", type.c_str());
   }
-  auto cpu_cores = seastar::resource::parse_cpuset(
-    get_conf<std::string>("crimson_alien_thread_cpu_cores"));
-  //  crimson_alien_thread_cpu_cores are assigned to alien threads.
-  if (!cpu_cores.has_value()) {
-    // no core isolation by default, seastar_cpu_cores will be
-    // shared between both alien and seastar reactor threads.
-    cpu_cores = seastar::resource::parse_cpuset(
-      get_conf<std::string>("crimson_seastar_cpu_cores"));
-    ceph_assert(cpu_cores.has_value());
+  /*
+   * crimson_alien_thread_cpu_cores must be set for optimal performance.
+   * Otherwise, no CPU pinning will take place.
+  */
+  std::optional<seastar::resource::cpuset> alien_thread_cpu_cores;
+
+  if (std::string conf_cpu_cores =
+        get_conf<std::string>("crimson_alien_thread_cpu_cores");
+      !conf_cpu_cores.empty()) {
+    logger().debug("{} using crimson_alien_thread_cpu_cores", __func__);
+    alien_thread_cpu_cores =
+      seastar::resource::parse_cpuset(conf_cpu_cores);
   }
+
   const auto num_threads =
     get_conf<uint64_t>("crimson_alien_op_num_threads");
-  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, cpu_cores);
+  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, alien_thread_cpu_cores);
   return tp->start();
 }
 

diff --git a/src/crimson/osd/main_config_bootstrap_helpers.cc b/src/crimson/osd/main_config_bootstrap_helpers.cc
index cbb22ec0e6b06c9009e952652c58fb863c0b33cc..c4e7fb72e4782ce3bf53b1016880e00c6db3bcc7 100644 (file)
--- a/src/crimson/osd/main_config_bootstrap_helpers.cc
+++ b/src/crimson/osd/main_config_bootstrap_helpers.cc
@@ -150,16 +150,32 @@ _get_early_config(int argc, const char *argv[])
              std::end(early_args),
              [](auto* arg) { return "--cpuset"sv == arg; });
            found == std::end(early_args)) {
-         auto smp_config = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
-         if (!smp_config.empty()) {
+         auto cpu_cores = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
+         if (!cpu_cores.empty()) {
            // Set --cpuset based on crimson_seastar_cpu_cores config option
            // --smp default is one per CPU
            ret.early_args.emplace_back("--cpuset");
-           ret.early_args.emplace_back(smp_config);
-           logger().info("get_early_config: set --cpuset {}", smp_config);
+           ret.early_args.emplace_back(cpu_cores);
+           ret.early_args.emplace_back("--thread-affinity");
+           ret.early_args.emplace_back("1");
+           logger().info("get_early_config: set --thread-affinity 1 --cpuset {}",
+                         cpu_cores);
          } else {
-           logger().warn("get_early_config: no cpuset specified, falling back"
-                         " to seastar's default of: all");
+           auto reactor_num = crimson::common::get_conf<uint64_t>("crimson_seastar_num_threads");
+           if (!reactor_num) {
+             logger().error("get_early_config: crimson_seastar_cpu_cores"
+                             " or crimson_seastar_num_threads"
+                             " must be set");
+             ceph_abort();
+           }
+           std::string smp = fmt::format("{}", reactor_num);
+           ret.early_args.emplace_back("--smp");
+           ret.early_args.emplace_back(smp);
+           ret.early_args.emplace_back("--thread-affinity");
+           ret.early_args.emplace_back("0");
+           logger().info("get_early_config: set --thread-affinity 0 --smp {}",
+                         smp);
+
          }
        } else {
          logger().error("get_early_config: --cpuset can be "