crimson: introduce crimson_seastar_cpu_cores

seastar::smp::count is deduced from the cpuset provided.

from seastar/src/core/reactor:
```
   if (smp_opts.smp) {
        smp::count = smp_opts.smp.get_value();
    } else {
        smp::count = cpu_set.size();
    }
```

Co-authored-by: Yingxin Cheng <yingxin.cheng@intel.com>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>

diff --git a/qa/config/crimson_qa_overrides.yaml b/qa/config/crimson_qa_overrides.yaml
index 8720103dab6203d16538514c99065e6826aa1c6a..fa8f49a498683f9da94107295683e575d5edc912 100644 (file)
--- a/qa/config/crimson_qa_overrides.yaml
+++ b/qa/config/crimson_qa_overrides.yaml
@@ -8,7 +8,6 @@ overrides:
       mon:
         osd pool default crimson: true
       osd:
-        crimson seastar smp: 3
         crimson osd obc lru size: 10
     flavor: crimson
   workunit:

diff --git a/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml b/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml
index 9774de6887bb453f7710bf232a2f229a40388df4..79641f695ab67c14a6135f97f1e4d6a5e4b50ebf 100644 (file)
--- a/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml
+++ b/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml
@@ -6,6 +6,15 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
       global:
         ms cluster mode: crc
         ms service mode: crc

diff --git a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
index 3f322f371496c6579c3c0de9c5b04c146aeefe28..3409ab6d63454172001243fbba4badd547d784e4 100644 (file)
--- a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
+++ b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
@@ -10,6 +10,15 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
       global:
         ms cluster mode: crc
         ms service mode: crc

diff --git a/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml b/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml
index d204f3eb2d0e779941abfb56091ce0f9ebc11c17..2bde4afcda5d217bd2848829c19edb1e4a735b30 100644 (file)
--- a/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml
+++ b/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml
@@ -10,6 +10,13 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-1
+      osd.1:
+        crimson seastar cpu cores: 2-3
+      osd.2:
+        crimson seastar cpu cores: 4-5
       global:
         ms cluster mode: crc
         ms service mode: crc

diff --git a/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml b/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml
index f475d5dc39b7c7a326f401999415012cf6482de3..56e4b7d657b875ac45013021d900619530ac053b 100644 (file)
--- a/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml
+++ b/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml
@@ -26,4 +26,12 @@ tasks:
     conf:
       osd:
         osd min pg log entries: 5
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-1
+      osd.1:
+        crimson seastar cpu cores: 2-3
+      osd.2:
+        crimson seastar cpu cores: 4-5
+      global:
 - osd_backfill:

diff --git a/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml b/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml
index 9774de6887bb453f7710bf232a2f229a40388df4..79641f695ab67c14a6135f97f1e4d6a5e4b50ebf 100644 (file)
--- a/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml
+++ b/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml
@@ -6,6 +6,15 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
       global:
         ms cluster mode: crc
         ms service mode: crc

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index d3c2c57a674c832c2dd56cf5f3cc0fdbaea17567..c52c54d5250b65764e616df12e10fcf3f4112062 100644 (file)
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -19,17 +19,18 @@ options:
   default: 6
   flags:
   - startup
-- name: crimson_seastar_smp
-  type: uint
+- name: crimson_seastar_cpu_cores
+  type: str
   level: advanced
-  desc: Number of seastar reactor threads to use for the osd
-  default: 1
+  desc: CPU cores on which seastar reactor threads will run in cpuset(7) format, smp::count is deduced from this option
   flags:
   - startup
 - name: crimson_alien_thread_cpu_cores
   type: str
   level: advanced
   desc: CPU cores on which alienstore threads will run in cpuset(7) format
+  flags:
+  - startup
 - name: seastore_segment_size
   type: size
   desc: Segment size to use for SegmentManager

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 31dedbcf893fbb9a514d3e10c36ebea02e1ffc04..32696775bb09271bd09fdeb2bf375003d275975e 100644 (file)
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -105,17 +105,11 @@ seastar::future<> AlienStore::start()
     get_conf<std::string>("crimson_alien_thread_cpu_cores"));
   //  crimson_alien_thread_cpu_cores are assigned to alien threads.
   if (!cpu_cores.has_value()) {
-    // no core isolation by default, cores [0, seastar::smp::count)
-    // will be shared between both alien and seastar reactor threads.
-    seastar::resource::cpuset cpuset;
-    std::copy(boost::counting_iterator<unsigned>(0),
-             boost::counting_iterator<unsigned>(seastar::smp::count - 1),
-             std::inserter(cpuset, cpuset.end()));
-    if (cpuset.empty()) {
-      logger().error("{}: unable to get nproc: {}", __func__, errno);
-    } else {
-      cpu_cores = cpuset;
-    }
+    // no core isolation by default, seastar_cpu_cores will be
+    // shared between both alien and seastar reactor threads.
+    cpu_cores = seastar::resource::parse_cpuset(
+      get_conf<std::string>("crimson_seastar_cpu_cores"));
+    ceph_assert(cpu_cores.has_value());
   }
   const auto num_threads =
     get_conf<uint64_t>("crimson_alien_op_num_threads");

diff --git a/src/crimson/osd/main_config_bootstrap_helpers.cc b/src/crimson/osd/main_config_bootstrap_helpers.cc
index 807fd1591aed5c4c7cf8e240d7b1b74f9f28d8a9..cbb22ec0e6b06c9009e952652c58fb863c0b33cc 100644 (file)
--- a/src/crimson/osd/main_config_bootstrap_helpers.cc
+++ b/src/crimson/osd/main_config_bootstrap_helpers.cc
@@ -148,17 +148,23 @@ _get_early_config(int argc, const char *argv[])
        if (auto found = std::find_if(
              std::begin(early_args),
              std::end(early_args),
-             [](auto* arg) { return "--smp"sv == arg; });
+             [](auto* arg) { return "--cpuset"sv == arg; });
            found == std::end(early_args)) {
-
-         // Set --smp based on crimson_seastar_smp config option
-         ret.early_args.emplace_back("--smp");
-
-         auto smp_config = local_conf().get_val<uint64_t>(
-           "crimson_seastar_smp");
-
-         ret.early_args.emplace_back(fmt::format("{}", smp_config));
-         logger().info("get_early_config: set --smp {}", smp_config);
+         auto smp_config = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
+         if (!smp_config.empty()) {
+           // Set --cpuset based on crimson_seastar_cpu_cores config option
+           // --smp default is one per CPU
+           ret.early_args.emplace_back("--cpuset");
+           ret.early_args.emplace_back(smp_config);
+           logger().info("get_early_config: set --cpuset {}", smp_config);
+         } else {
+           logger().warn("get_early_config: no cpuset specified, falling back"
+                         " to seastar's default of: all");
+         }
+       } else {
+         logger().error("get_early_config: --cpuset can be "
+                        "set only using crimson_seastar_cpu_cores");
+         ceph_abort();
        }
        return 0;
       });

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 063f9d86ea7799f74b1a86b6849b7e5c1636d122..5c02b2895c2d9f3240cce2dbb24285a80f7f342b 100644 (file)
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -362,7 +362,7 @@ namespace {
 seastar::future<> OSD::start()
 {
   LOG_PREFIX(OSD::start);
-  INFO();
+  INFO("seastar::smp::count {}", seastar::smp::count);
 
   startup_time = ceph::mono_clock::now();
   ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);

diff --git a/src/vstart.sh b/src/vstart.sh
index f9b2f3ce4cf2198afb3b6098b80fedeadb8eeb83..2a0e2a615a13eef8561fb0a0d646a86a3d6029dd 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -1110,18 +1110,12 @@ start_osd() {
     local osds_wait
     for osd in `seq $start $end`
     do
-       local extra_seastar_args
        if [ "$ceph_osd" == "crimson-osd" ]; then
         bottom_cpu=$(( osd * crimson_smp ))
         top_cpu=$(( bottom_cpu + crimson_smp - 1 ))
-           # set a single CPU nodes for each osd
-           extra_seastar_args="--cpuset $bottom_cpu-$top_cpu"
-           if [ "$debug" -ne 0 ]; then
-               extra_seastar_args+=" --debug"
-           fi
-            if [ "$trace" -ne 0 ]; then
-                extra_seastar_args+=" --trace"
-            fi
+           # set exclusive CPU nodes for each osd
+           echo "$CEPH_BIN/ceph -c $conf_fn config set osd.$osd crimson_seastar_cpu_cores $bottom_cpu-$top_cpu"
+           $CEPH_BIN/ceph -c $conf_fn config set "osd.$osd" crimson_seastar_cpu_cores "$bottom_cpu-$top_cpu"
        fi
        if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then
             wconf <<EOF
@@ -1665,7 +1659,12 @@ EOF
 fi
 
 if [ "$ceph_osd" == "crimson-osd" ]; then
-    $CEPH_BIN/ceph -c $conf_fn config set osd crimson_seastar_smp $crimson_smp
+     if [ "$debug" -ne 0 ]; then
+        extra_seastar_args=" --debug"
+    fi
+    if [ "$trace" -ne 0 ]; then
+        extra_seastar_args=" --trace"
+    fi
     if [ "$(expr $(nproc) - 1)" -gt "$(($CEPH_NUM_OSD * crimson_smp))" ]; then
       echo "crimson_alien_thread_cpu_cores:" $(($CEPH_NUM_OSD * crimson_smp))-"$(expr $(nproc) - 1)"
       $CEPH_BIN/ceph -c $conf_fn config set osd crimson_alien_thread_cpu_cores $(($CEPH_NUM_OSD * crimson_smp))-"$(expr $(nproc) - 1)"