From c35ac40e0b5addcac21941c505f24b4717cca8c3 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Thu, 3 Jan 2019 17:24:46 -0600
Subject: [PATCH] os/bluestore: report numa node info for storage

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/os/ObjectStore.h           |  8 +++++
 src/os/bluestore/BlockDevice.h |  3 ++
 src/os/bluestore/BlueStore.cc  | 59 ++++++++++++++++++++++++++++++++++
 src/os/bluestore/BlueStore.h   |  2 --
 4 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index 7cb53adf448..86e070de6c3 100644
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -1546,6 +1546,14 @@ public:
     return is_rotational() ? "hdd" : "ssd";
   }
 
+  virtual int get_numa_node(
+    int *numa_node,
+    set<int> *nodes,
+    set<string> *failed) {
+    return -EOPNOTSUPP;
+  }
+
+
   virtual bool can_sort_nibblewise() {
     return false;   // assume a backend cannot, unless it says otherwise
   }
diff --git a/src/os/bluestore/BlockDevice.h b/src/os/bluestore/BlockDevice.h
index 645028acc7d..07fd3e2168b 100644
--- a/src/os/bluestore/BlockDevice.h
+++ b/src/os/bluestore/BlockDevice.h
@@ -182,6 +182,9 @@ public:
     }
     return 0;
   }
+  virtual int get_numa_node(int *node) const {
+    return -EOPNOTSUPP;
+  }
 
   virtual int read(
     uint64_t off,
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index cdb922d16c6..e7f422d0b87 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -36,6 +36,8 @@
 #include "auth/Crypto.h"
 #include "common/EventTrace.h"
 #include "perfglue/heap_profiler.h"
+#include "common/blkdev.h"
+#include "common/numa.h"
 
 #define dout_context cct
 #define dout_subsys ceph_subsys_bluestore
@@ -7772,6 +7774,63 @@ void BlueStore::collect_metadata(map<string,string> *pm)
   } else {
     (*pm)["bluefs"] = "0";
   }
+
+  // report numa mapping for underlying devices
+  int node = -1;
+  set<int> nodes;
+  set<string> failed;
+  int r = get_numa_node(&node, &nodes, &failed);
+  if (r >= 0) {
+    if (!failed.empty()) {
+      (*pm)["objectstore_numa_unknown_devices"] = stringify(failed);
+    }
+    if (!nodes.empty()) {
+      dout(1) << __func__ << " devices span numa nodes " << nodes << dendl;
+      (*pm)["objectstore_numa_nodes"] = stringify(nodes);
+    }
+    if (node >= 0) {
+      (*pm)["objectstore_numa_node"] = stringify(node);
+    }
+  }
+}
+
+int BlueStore::get_numa_node(
+  int *final_node,
+  set<int> *out_nodes,
+  set<string> *out_failed)
+{
+  int node = -1;
+  set<string> devices;
+  get_devices(&devices);
+  set<int> nodes;
+  set<string> failed;
+  for (auto& devname : devices) {
+    int n;
+    BlkDev bdev(devname);
+    int r = bdev.get_numa_node(&n);
+    if (r < 0) {
+      dout(10) << __func__ << " bdev " << devname << " can't detect numa_node"
+	       << dendl;
+      failed.insert(devname);
+      continue;
+    }
+    dout(10) << __func__ << " bdev " << devname << " on numa_node " << n
+	     << dendl;
+    nodes.insert(n);
+    if (node < 0) {
+      node = n;
+    }
+  }
+  if (node >= 0 && nodes.size() == 1 && failed.empty()) {
+    *final_node = node;
+  }
+  if (out_nodes) {
+    *out_nodes = nodes;
+  }
+  if (out_failed) {
+    *out_failed = failed;
+  }
+  return 0;
 }
 
 int BlueStore::get_devices(set<string> *ls)
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 42f5cbb051f..8cb69d6330f 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2362,8 +2362,6 @@ public:
 
   int get_numa_node(
     int *numa_node,
-    size_t *cpu_set_size,
-    cpu_set_t *cpu_set,
     set<int> *nodes,
     set<string> *failed) override;
 
-- 
2.39.5