From: Matan Breizman Date: Tue, 21 May 2024 09:56:01 +0000 (+0000) Subject: crimson: introduce crimson_seastar_num_threads X-Git-Tag: v19.1.1~227^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=9e6bc63eb97e9573da260dd48cdf351a39f6379b;p=ceph.git crimson: introduce crimson_seastar_num_threads This new option is there to allow *basic* cluster deployments. crimson_seastar_num_threads can be used globally for all the OSDs in the cluster: ``` osd: crimson_seastar_num_threads: crimson_alien_op_num_threads: ``` As a result, all the available CPUs will be allocated to *both* seastar reactor threads and to Alienstore threads - without any exclusion. Notes: * The core allocation will most likely overlap between OSDS and/or Seastar and Alienstore. An optiomal deployment requires `crimson_alien_thread_cpu_cores` and `crimson_seastar_cpu_cores` to be set for each OSD based on the host its located at. * Seastar reactor number (smp::count) will be deduced directly from crimson_seastar_num_threads --- Documentation followup: https://github.com/ceph/ceph/pull/57352 --- ### Option 1 (Optimal) * CPUs are pinned to the provided sets (seastar and alien cpu sets) * thread-affinity = 1 (seastar's default). * smp (smp::count / reactor count): `seastar_cpu_set.size()` * Seastar reactor threads: `seastar_cpu_set.size()` * Alienstore threads num: `crimson_alien_op_num_threads` --- ### Option 2 (Basic): * No CPU pinning. * thread-affinity = 0 * smp (smp::count / reactor count) = `crimson_seastar_num_threads` * Seastar reactor threads num: `crimson_seastar_num_threads` * Alienstore threads num: `crimson_alien_op_num_threads` --- Fixes: https://tracker.ceph.com/issues/65752 Signed-off-by: Matan Breizman (cherry picked from commit e0b6fff4f3bf9495fc0fdc17e5ea1f53ffa1d423) --- diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index 7030780d9a9fd..32b6873f0c233 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -31,6 +31,15 @@ options: desc: CPU cores on which alienstore threads will run in cpuset(7) format flags: - startup +- name: crimson_seastar_num_threads + type: uint + level: advanced + default: 0 + desc: The number of threads for serving seastar reactors without CPU pinning, overridden if crimson_seastar_cpu_cores is set + flags: + - startup + min: 0 + max: 32 - name: crimson_osd_stat_interval type: int level: advanced diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc index 2139a2dd25856..2fac9a89ee4ef 100644 --- a/src/crimson/os/alienstore/alien_store.cc +++ b/src/crimson/os/alienstore/alien_store.cc @@ -101,19 +101,23 @@ seastar::future<> AlienStore::start() if (!store) { ceph_abort_msgf("unsupported objectstore type: %s", type.c_str()); } - auto cpu_cores = seastar::resource::parse_cpuset( - get_conf("crimson_alien_thread_cpu_cores")); - // crimson_alien_thread_cpu_cores are assigned to alien threads. - if (!cpu_cores.has_value()) { - // no core isolation by default, seastar_cpu_cores will be - // shared between both alien and seastar reactor threads. - cpu_cores = seastar::resource::parse_cpuset( - get_conf("crimson_seastar_cpu_cores")); - ceph_assert(cpu_cores.has_value()); + /* + * crimson_alien_thread_cpu_cores must be set for optimal performance. + * Otherwise, no CPU pinning will take place. + */ + std::optional alien_thread_cpu_cores; + + if (std::string conf_cpu_cores = + get_conf("crimson_alien_thread_cpu_cores"); + !conf_cpu_cores.empty()) { + logger().debug("{} using crimson_alien_thread_cpu_cores", __func__); + alien_thread_cpu_cores = + seastar::resource::parse_cpuset(conf_cpu_cores); } + const auto num_threads = get_conf("crimson_alien_op_num_threads"); - tp = std::make_unique(num_threads, 128, cpu_cores); + tp = std::make_unique(num_threads, 128, alien_thread_cpu_cores); return tp->start(); } diff --git a/src/crimson/osd/main_config_bootstrap_helpers.cc b/src/crimson/osd/main_config_bootstrap_helpers.cc index cbb22ec0e6b06..c4e7fb72e4782 100644 --- a/src/crimson/osd/main_config_bootstrap_helpers.cc +++ b/src/crimson/osd/main_config_bootstrap_helpers.cc @@ -150,16 +150,32 @@ _get_early_config(int argc, const char *argv[]) std::end(early_args), [](auto* arg) { return "--cpuset"sv == arg; }); found == std::end(early_args)) { - auto smp_config = crimson::common::get_conf("crimson_seastar_cpu_cores"); - if (!smp_config.empty()) { + auto cpu_cores = crimson::common::get_conf("crimson_seastar_cpu_cores"); + if (!cpu_cores.empty()) { // Set --cpuset based on crimson_seastar_cpu_cores config option // --smp default is one per CPU ret.early_args.emplace_back("--cpuset"); - ret.early_args.emplace_back(smp_config); - logger().info("get_early_config: set --cpuset {}", smp_config); + ret.early_args.emplace_back(cpu_cores); + ret.early_args.emplace_back("--thread-affinity"); + ret.early_args.emplace_back("1"); + logger().info("get_early_config: set --thread-affinity 1 --cpuset {}", + cpu_cores); } else { - logger().warn("get_early_config: no cpuset specified, falling back" - " to seastar's default of: all"); + auto reactor_num = crimson::common::get_conf("crimson_seastar_num_threads"); + if (!reactor_num) { + logger().error("get_early_config: crimson_seastar_cpu_cores" + " or crimson_seastar_num_threads" + " must be set"); + ceph_abort(); + } + std::string smp = fmt::format("{}", reactor_num); + ret.early_args.emplace_back("--smp"); + ret.early_args.emplace_back(smp); + ret.early_args.emplace_back("--thread-affinity"); + ret.early_args.emplace_back("0"); + logger().info("get_early_config: set --thread-affinity 0 --smp {}", + smp); + } } else { logger().error("get_early_config: --cpuset can be "