os/bluestore: allow multiple SPDK BlueStore OSD instances

Fixes: http://tracker.ceph.com/issues/16966
Signed-off-by: Orlando Moreno <orlando.moreno@intel.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 41cc525958a7f72d7881876d6cf0c886b410f97b..1def06be07dd7dab3e7d3c8f1b29d266eb23a6e0 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -999,6 +999,11 @@ OPTION(bluestore_bluefs_reclaim_ratio, OPT_FLOAT, .20) // how much to reclaim at
 // get the serial number of Intel(R) Fultondale NVMe controllers.
 // Example:
 // bluestore_block_path = spdk:55cd2e404bd73932
+// If you want to run multiple SPDK instances per node, you must specify the
+// amount of memory per socket each instance will use.
+OPTION(spdk_socket_mem, OPT_STR, "512,512")
+// A hexadecimal bit mask of the cores to run on. Note the core numbering can change between platforms and should be determined beforehand.
+OPTION(spdk_coremask, OPT_STR, "0x3")
 OPTION(bluestore_block_path, OPT_STR, "")
 OPTION(bluestore_block_size, OPT_U64, 10 * 1024*1024*1024)  // 10gb for testing
 OPTION(bluestore_block_create, OPT_BOOL, true)

diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc
index 13b01f0a20d6c969a1e9120b45d38d06920269ec..70c28144b85e396984b2e9ed21ea0658c9fa9933 100644 (file)
--- a/src/os/bluestore/NVMEDevice.cc
+++ b/src/os/bluestore/NVMEDevice.cc
@@ -626,6 +626,17 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver)
 {
   Mutex::Locker l(lock);
   int r = 0;
+
+  string prefix = "--file-prefix=";
+  string sock_mem = "--socket-mem=";
+  string coremask = "-c ";
+  prefix += sn_tag;
+  sock_mem += g_conf->spdk_socket_mem;
+  coremask += g_conf->spdk_coremask;
+  char *prefix_arg = (char *)prefix.c_str();
+  char *sock_mem_arg = (char *)sock_mem.c_str();
+  char *coremask_arg = (char *)coremask.c_str();
+
   if (sn_tag.empty()) {
     r = -ENOENT;
     derr << __func__ << " empty serial number: " << cpp_strerror(r) << dendl;
@@ -642,11 +653,13 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver)
   if (!init) {
     init = true;
     dpdk_thread = std::thread(
-      [this]() {
+      [this, prefix_arg, sock_mem_arg, coremask_arg]() {
         static const char *ealargs[] = {
             "ceph-osd",
-            "-c 0x3", /* This must be the second parameter. It is overwritten by index in main(). */
+            coremask_arg, /* This must be the second parameter. It is overwritten by index in main(). */
             "-n 4",
+           socket_mem_arg,
+           prefix_arg
         };
 
         int r = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), (char **)(void *)(uintptr_t)ealargs);