From 46684f2ffb3fd4463b489379ab3a5a44906ae2a0 Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Sat, 19 Oct 2019 14:27:06 +0800
Subject: [PATCH] rgw/rgw_main: auto set radosgw's cpu affinity according to
 numa_node configuration

Fixes: https://tracker.ceph.com/issues/42272
Signed-off-by: luo rixin <luorixin@huawei.com>
---
 src/common/options.cc |  5 +++++
 src/rgw/rgw_main.cc   | 22 ++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/common/options.cc b/src/common/options.cc
index 77d04ed6231d..cb33c982f2a0 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5788,6 +5788,11 @@ std::vector<Option> get_rgw_options() {
     .set_long_description(
         "This is needed for virtual hosting of buckets, unless configured via zonegroup "
         "configuration."),
+    
+    Option("rgw_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(-1)
+    .set_flag(Option::FLAG_STARTUP)
+    .set_description("set rgw's cpu affinity to a numa node (-1 for none)"),
 
     Option("rgw_service_provider_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("")
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index d8afd86cdf11..2d6d00368054 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -9,6 +9,7 @@
 #include "common/Timer.h"
 #include "common/safe_io.h"
 #include "common/TracepointProvider.h"
+#include "common/numa.h"
 #include "include/compat.h"
 #include "include/str_list.h"
 #include "include/stringify.h"
@@ -252,6 +253,27 @@ int main(int argc, const char **argv)
 			 CODE_ENVIRONMENT_DAEMON,
 			 flags, "rgw_data", false);
 
+  int numa_node = g_conf().get_val<int64_t>("rgw_numa_node");
+  size_t numa_cpu_set_size = 0;
+  cpu_set_t numa_cpu_set;
+
+  if (numa_node >= 0) {
+    int r = get_numa_node_cpu_set(numa_node, &numa_cpu_set_size, &numa_cpu_set);
+    if (r < 0) {
+      dout(1) << __func__ << " unable to determine rgw numa node " << numa_node
+              << " CPUs" << dendl;
+      numa_node = -1;
+    } else {
+      r = set_cpu_affinity_all_threads(numa_cpu_set_size, &numa_cpu_set);
+      if (r < 0) {
+        derr << __func__ << " failed to set numa affinity: " << cpp_strerror(r)
+        << dendl;
+      }
+    }
+  } else {
+    dout(1) << __func__ << " not setting numa affinity" << dendl;
+  }
+
   // maintain existing region root pool for new multisite objects
   if (!g_conf()->rgw_region_root_pool.empty()) {
     const char *root_pool = g_conf()->rgw_region_root_pool.c_str();
-- 
2.47.3