os/bluestore: report numa node info for storage

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index 7cb53adf4486632c84b567d3500abc5492e7429a..86e070de6c389bb6f03cbbdc502262d46f8de2b3 100644 (file)
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -1546,6 +1546,14 @@ public:
     return is_rotational() ? "hdd" : "ssd";
   }
 
+  virtual int get_numa_node(
+    int *numa_node,
+    set<int> *nodes,
+    set<string> *failed) {
+    return -EOPNOTSUPP;
+  }
+
+
   virtual bool can_sort_nibblewise() {
     return false;   // assume a backend cannot, unless it says otherwise
   }

diff --git a/src/os/bluestore/BlockDevice.h b/src/os/bluestore/BlockDevice.h
index 645028acc7d7e65d76bbe0d5f9fd46265c97b0a3..07fd3e2168b525a6d61aa4d2739174623a68a7de 100644 (file)
--- a/src/os/bluestore/BlockDevice.h
+++ b/src/os/bluestore/BlockDevice.h
@@ -182,6 +182,9 @@ public:
     }
     return 0;
   }
+  virtual int get_numa_node(int *node) const {
+    return -EOPNOTSUPP;
+  }
 
   virtual int read(
     uint64_t off,

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index cdb922d16c624e54ad1fe766a9f30bd0cf3ccf4b..e7f422d0b87ac7fa00db785347637919c8aa7aca 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -36,6 +36,8 @@
 #include "auth/Crypto.h"
 #include "common/EventTrace.h"
 #include "perfglue/heap_profiler.h"
+#include "common/blkdev.h"
+#include "common/numa.h"
 
 #define dout_context cct
 #define dout_subsys ceph_subsys_bluestore
@@ -7772,6 +7774,63 @@ void BlueStore::collect_metadata(map<string,string> *pm)
   } else {
     (*pm)["bluefs"] = "0";
   }
+
+  // report numa mapping for underlying devices
+  int node = -1;
+  set<int> nodes;
+  set<string> failed;
+  int r = get_numa_node(&node, &nodes, &failed);
+  if (r >= 0) {
+    if (!failed.empty()) {
+      (*pm)["objectstore_numa_unknown_devices"] = stringify(failed);
+    }
+    if (!nodes.empty()) {
+      dout(1) << __func__ << " devices span numa nodes " << nodes << dendl;
+      (*pm)["objectstore_numa_nodes"] = stringify(nodes);
+    }
+    if (node >= 0) {
+      (*pm)["objectstore_numa_node"] = stringify(node);
+    }
+  }
+}
+
+int BlueStore::get_numa_node(
+  int *final_node,
+  set<int> *out_nodes,
+  set<string> *out_failed)
+{
+  int node = -1;
+  set<string> devices;
+  get_devices(&devices);
+  set<int> nodes;
+  set<string> failed;
+  for (auto& devname : devices) {
+    int n;
+    BlkDev bdev(devname);
+    int r = bdev.get_numa_node(&n);
+    if (r < 0) {
+      dout(10) << __func__ << " bdev " << devname << " can't detect numa_node"
+              << dendl;
+      failed.insert(devname);
+      continue;
+    }
+    dout(10) << __func__ << " bdev " << devname << " on numa_node " << n
+            << dendl;
+    nodes.insert(n);
+    if (node < 0) {
+      node = n;
+    }
+  }
+  if (node >= 0 && nodes.size() == 1 && failed.empty()) {
+    *final_node = node;
+  }
+  if (out_nodes) {
+    *out_nodes = nodes;
+  }
+  if (out_failed) {
+    *out_failed = failed;
+  }
+  return 0;
 }
 
 int BlueStore::get_devices(set<string> *ls)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 42f5cbb051fa112f59b4c5012e35e24028b99294..8cb69d6330ff03db395b743533c96dad295f4a07 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2362,8 +2362,6 @@ public:
 
   int get_numa_node(
     int *numa_node,
-    size_t *cpu_set_size,
-    cpu_set_t *cpu_set,
     set<int> *nodes,
     set<string> *failed) override;