rgw/rgw_main: auto set radosgw's cpu affinity according to numa_node configuration

Fixes: https://tracker.ceph.com/issues/42272
Signed-off-by: luo rixin <luorixin@huawei.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index 77d04ed6231d5890b447c249c59c6cf5966c7d99..cb33c982f2a08e0a8b6324ae8c824f034fb93027 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5788,6 +5788,11 @@ std::vector<Option> get_rgw_options() {
     .set_long_description(
         "This is needed for virtual hosting of buckets, unless configured via zonegroup "
         "configuration."),
+    
+    Option("rgw_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(-1)
+    .set_flag(Option::FLAG_STARTUP)
+    .set_description("set rgw's cpu affinity to a numa node (-1 for none)"),
 
     Option("rgw_service_provider_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("")

diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index d8afd86cdf1150111a40b75eb8e117ba7ea1258d..2d6d00368054443718ec7c631df7abfd6528d4c8 100644 (file)
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -9,6 +9,7 @@
 #include "common/Timer.h"
 #include "common/safe_io.h"
 #include "common/TracepointProvider.h"
+#include "common/numa.h"
 #include "include/compat.h"
 #include "include/str_list.h"
 #include "include/stringify.h"
@@ -252,6 +253,27 @@ int main(int argc, const char **argv)
                         CODE_ENVIRONMENT_DAEMON,
                         flags, "rgw_data", false);
 
+  int numa_node = g_conf().get_val<int64_t>("rgw_numa_node");
+  size_t numa_cpu_set_size = 0;
+  cpu_set_t numa_cpu_set;
+
+  if (numa_node >= 0) {
+    int r = get_numa_node_cpu_set(numa_node, &numa_cpu_set_size, &numa_cpu_set);
+    if (r < 0) {
+      dout(1) << __func__ << " unable to determine rgw numa node " << numa_node
+              << " CPUs" << dendl;
+      numa_node = -1;
+    } else {
+      r = set_cpu_affinity_all_threads(numa_cpu_set_size, &numa_cpu_set);
+      if (r < 0) {
+        derr << __func__ << " failed to set numa affinity: " << cpp_strerror(r)
+        << dendl;
+      }
+    }
+  } else {
+    dout(1) << __func__ << " not setting numa affinity" << dendl;
+  }
+
   // maintain existing region root pool for new multisite objects
   if (!g_conf()->rgw_region_root_pool.empty()) {
     const char *root_pool = g_conf()->rgw_region_root_pool.c_str();