fs/ceph_mds: set ceph_mds cpu affinity

fs/ceph_mds: set ceph_mds cpu affinity according to mds_numa_node config option

Signed-off-by: qilianghong <qilianghong@huawei.com>

diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc
index 38e673755a95a470dba6a303ffe5547e0324da23..41adf75f131047b8ad38e7221d021717d4d501be 100644 (file)
--- a/src/ceph_mds.cc
+++ b/src/ceph_mds.cc
@@ -26,6 +26,7 @@
 
 #include "common/config.h"
 #include "common/strtol.h"
+#include "common/numa.h"
 
 #include "mon/MonMap.h"
 #include "mds/MDSDaemon.h"
@@ -93,6 +94,25 @@ int main(int argc, const char **argv)
                         0, "mds_data");
   ceph_heap_profiler_init();
 
+  int numa_node = g_conf().get_val<int64_t>("mds_numa_node");
+  size_t numa_cpu_set_size = 0;
+  cpu_set_t numa_cpu_set;
+  if (numa_node >= 0) {
+    int r = get_numa_node_cpu_set(numa_node, &numa_cpu_set_size, &numa_cpu_set);
+    if (r < 0) {
+      dout(1) << __func__ << " unable to determine mds numa node " << numa_node
+              << " CPUs" << dendl;
+      numa_node = -1;
+    } else {
+      r = set_cpu_affinity_all_threads(numa_cpu_set_size, &numa_cpu_set);
+      if (r < 0) {
+        derr << __func__ << " failed to set numa affinity: " << cpp_strerror(r)
+        << dendl;
+      }
+    }
+  } else {
+    dout(1) << __func__ << " not setting numa affinity" << dendl;
+  }
   std::string val, action;
   for (std::vector<const char*>::iterator i = args.begin(); i != args.end(); ) {
     if (ceph_argparse_double_dash(args, i)) {

diff --git a/src/common/options.cc b/src/common/options.cc
index f20568501cf2ded256a1f07a5abdfcbf6f018346..19deaca500f0d5a6ddac8eca96336c74c8e66297 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7413,6 +7413,11 @@ static std::vector<Option> get_immutable_object_cache_options() {
 
 std::vector<Option> get_mds_options() {
   return std::vector<Option>({
+    Option("mds_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(-1)
+    .set_flag(Option::FLAG_STARTUP)
+    .set_description("set mds's cpu affinity to a numa node (-1 for none)"),
+
     Option("mds_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("/var/lib/ceph/mds/$cluster-$id")
     .set_flag(Option::FLAG_NO_MON_UPDATE)