From e547768fedd02d631dc59cd32e9f026c1743fd31 Mon Sep 17 00:00:00 2001 From: Matan Breizman Date: Tue, 30 Jan 2024 17:53:16 +0000 Subject: [PATCH] crimson: introduce crimson_seastar_cpu_cores seastar::smp::count is deduced from the cpuset provided. from seastar/src/core/reactor: ``` if (smp_opts.smp) { smp::count = smp_opts.smp.get_value(); } else { smp::count = cpu_set.size(); } ``` Co-authored-by: Yingxin Cheng Signed-off-by: Matan Breizman --- qa/config/crimson_qa_overrides.yaml | 1 - .../crimson-rados/basic/clusters/fixed-2.yaml | 9 +++++++ .../crimson-rados/perf/clusters/fixed-2.yaml | 9 +++++++ .../crimson-rados/rbd/clusters/fixed-1.yaml | 7 +++++ .../singleton/all/osd-backfill.yaml | 8 ++++++ .../thrash/clusters/fixed-2.yaml | 9 +++++++ src/common/options/crimson.yaml.in | 9 ++++--- src/crimson/os/alienstore/alien_store.cc | 16 ++++-------- .../osd/main_config_bootstrap_helpers.cc | 26 ++++++++++++------- src/crimson/osd/osd.cc | 2 +- src/vstart.sh | 19 +++++++------- 11 files changed, 78 insertions(+), 37 deletions(-) diff --git a/qa/config/crimson_qa_overrides.yaml b/qa/config/crimson_qa_overrides.yaml index 8720103dab620..fa8f49a498683 100644 --- a/qa/config/crimson_qa_overrides.yaml +++ b/qa/config/crimson_qa_overrides.yaml @@ -8,7 +8,6 @@ overrides: mon: osd pool default crimson: true osd: - crimson seastar smp: 3 crimson osd obc lru size: 10 flavor: crimson workunit: diff --git a/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml b/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml index 9774de6887bb4..79641f695ab67 100644 --- a/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml +++ b/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml @@ -6,6 +6,15 @@ overrides: conf: osd: osd shutdown pgref assert: true + crimson alien thread cpu cores: 6-7 + osd.0: + crimson seastar cpu cores: 0-2 + osd.1: + crimson seastar cpu cores: 3-5 + osd.2: + crimson seastar cpu cores: 0-2 + osd.3: + crimson seastar cpu cores: 3-5 global: ms cluster mode: crc ms service mode: crc diff --git a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml index 3f322f371496c..3409ab6d63454 100644 --- a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml +++ b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml @@ -10,6 +10,15 @@ overrides: conf: osd: osd shutdown pgref assert: true + crimson alien thread cpu cores: 6-7 + osd.0: + crimson seastar cpu cores: 0-2 + osd.1: + crimson seastar cpu cores: 3-5 + osd.2: + crimson seastar cpu cores: 0-2 + osd.3: + crimson seastar cpu cores: 3-5 global: ms cluster mode: crc ms service mode: crc diff --git a/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml b/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml index d204f3eb2d0e7..2bde4afcda5d2 100644 --- a/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml +++ b/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml @@ -10,6 +10,13 @@ overrides: conf: osd: osd shutdown pgref assert: true + crimson alien thread cpu cores: 6-7 + osd.0: + crimson seastar cpu cores: 0-1 + osd.1: + crimson seastar cpu cores: 2-3 + osd.2: + crimson seastar cpu cores: 4-5 global: ms cluster mode: crc ms service mode: crc diff --git a/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml b/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml index f475d5dc39b7c..56e4b7d657b87 100644 --- a/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml +++ b/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml @@ -26,4 +26,12 @@ tasks: conf: osd: osd min pg log entries: 5 + crimson alien thread cpu cores: 6-7 + osd.0: + crimson seastar cpu cores: 0-1 + osd.1: + crimson seastar cpu cores: 2-3 + osd.2: + crimson seastar cpu cores: 4-5 + global: - osd_backfill: diff --git a/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml b/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml index 9774de6887bb4..79641f695ab67 100644 --- a/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml +++ b/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml @@ -6,6 +6,15 @@ overrides: conf: osd: osd shutdown pgref assert: true + crimson alien thread cpu cores: 6-7 + osd.0: + crimson seastar cpu cores: 0-2 + osd.1: + crimson seastar cpu cores: 3-5 + osd.2: + crimson seastar cpu cores: 0-2 + osd.3: + crimson seastar cpu cores: 3-5 global: ms cluster mode: crc ms service mode: crc diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index d3c2c57a674c8..c52c54d5250b6 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -19,17 +19,18 @@ options: default: 6 flags: - startup -- name: crimson_seastar_smp - type: uint +- name: crimson_seastar_cpu_cores + type: str level: advanced - desc: Number of seastar reactor threads to use for the osd - default: 1 + desc: CPU cores on which seastar reactor threads will run in cpuset(7) format, smp::count is deduced from this option flags: - startup - name: crimson_alien_thread_cpu_cores type: str level: advanced desc: CPU cores on which alienstore threads will run in cpuset(7) format + flags: + - startup - name: seastore_segment_size type: size desc: Segment size to use for SegmentManager diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc index 31dedbcf893fb..32696775bb092 100644 --- a/src/crimson/os/alienstore/alien_store.cc +++ b/src/crimson/os/alienstore/alien_store.cc @@ -105,17 +105,11 @@ seastar::future<> AlienStore::start() get_conf("crimson_alien_thread_cpu_cores")); // crimson_alien_thread_cpu_cores are assigned to alien threads. if (!cpu_cores.has_value()) { - // no core isolation by default, cores [0, seastar::smp::count) - // will be shared between both alien and seastar reactor threads. - seastar::resource::cpuset cpuset; - std::copy(boost::counting_iterator(0), - boost::counting_iterator(seastar::smp::count - 1), - std::inserter(cpuset, cpuset.end())); - if (cpuset.empty()) { - logger().error("{}: unable to get nproc: {}", __func__, errno); - } else { - cpu_cores = cpuset; - } + // no core isolation by default, seastar_cpu_cores will be + // shared between both alien and seastar reactor threads. + cpu_cores = seastar::resource::parse_cpuset( + get_conf("crimson_seastar_cpu_cores")); + ceph_assert(cpu_cores.has_value()); } const auto num_threads = get_conf("crimson_alien_op_num_threads"); diff --git a/src/crimson/osd/main_config_bootstrap_helpers.cc b/src/crimson/osd/main_config_bootstrap_helpers.cc index 807fd1591aed5..cbb22ec0e6b06 100644 --- a/src/crimson/osd/main_config_bootstrap_helpers.cc +++ b/src/crimson/osd/main_config_bootstrap_helpers.cc @@ -148,17 +148,23 @@ _get_early_config(int argc, const char *argv[]) if (auto found = std::find_if( std::begin(early_args), std::end(early_args), - [](auto* arg) { return "--smp"sv == arg; }); + [](auto* arg) { return "--cpuset"sv == arg; }); found == std::end(early_args)) { - - // Set --smp based on crimson_seastar_smp config option - ret.early_args.emplace_back("--smp"); - - auto smp_config = local_conf().get_val( - "crimson_seastar_smp"); - - ret.early_args.emplace_back(fmt::format("{}", smp_config)); - logger().info("get_early_config: set --smp {}", smp_config); + auto smp_config = crimson::common::get_conf("crimson_seastar_cpu_cores"); + if (!smp_config.empty()) { + // Set --cpuset based on crimson_seastar_cpu_cores config option + // --smp default is one per CPU + ret.early_args.emplace_back("--cpuset"); + ret.early_args.emplace_back(smp_config); + logger().info("get_early_config: set --cpuset {}", smp_config); + } else { + logger().warn("get_early_config: no cpuset specified, falling back" + " to seastar's default of: all"); + } + } else { + logger().error("get_early_config: --cpuset can be " + "set only using crimson_seastar_cpu_cores"); + ceph_abort(); } return 0; }); diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 063f9d86ea779..5c02b2895c2d9 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -362,7 +362,7 @@ namespace { seastar::future<> OSD::start() { LOG_PREFIX(OSD::start); - INFO(); + INFO("seastar::smp::count {}", seastar::smp::count); startup_time = ceph::mono_clock::now(); ceph_assert(seastar::this_shard_id() == PRIMARY_CORE); diff --git a/src/vstart.sh b/src/vstart.sh index f9b2f3ce4cf21..2a0e2a615a13e 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -1110,18 +1110,12 @@ start_osd() { local osds_wait for osd in `seq $start $end` do - local extra_seastar_args if [ "$ceph_osd" == "crimson-osd" ]; then bottom_cpu=$(( osd * crimson_smp )) top_cpu=$(( bottom_cpu + crimson_smp - 1 )) - # set a single CPU nodes for each osd - extra_seastar_args="--cpuset $bottom_cpu-$top_cpu" - if [ "$debug" -ne 0 ]; then - extra_seastar_args+=" --debug" - fi - if [ "$trace" -ne 0 ]; then - extra_seastar_args+=" --trace" - fi + # set exclusive CPU nodes for each osd + echo "$CEPH_BIN/ceph -c $conf_fn config set osd.$osd crimson_seastar_cpu_cores $bottom_cpu-$top_cpu" + $CEPH_BIN/ceph -c $conf_fn config set "osd.$osd" crimson_seastar_cpu_cores "$bottom_cpu-$top_cpu" fi if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then wconf <