NVMEDevice: Fix the issue of multiple OSDs when using SPDK

For each OSD when using SPDK, it should be the type of primary
process. So when there are multiple NVMe SSDs, each OSD will use
one. So we do not want an OSD process claim all the NVMe SSDs,
so we use the allowed list to restrict the probe.

And with this patch, we can support multiple OSDs while using
SPDK.

Signed-off-by: Ziye Yang <ziye.yang@intel.com>

diff --git a/src/blk/spdk/NVMEDevice.cc b/src/blk/spdk/NVMEDevice.cc
index 69ae45acf1210859904e806808d41241ed0bf80a..adf9e31abe71f3d247709a704442931dffcbb215 100644 (file)
--- a/src/blk/spdk/NVMEDevice.cc
+++ b/src/blk/spdk/NVMEDevice.cc
@@ -555,6 +555,12 @@ int NVMEManager::try_get(const spdk_nvme_transport_id& trid, SharedDriverData **
     }
   }
 
+  struct spdk_pci_addr pci_addr;
+  int rc = spdk_pci_addr_parse(&pci_addr, trid.traddr);
+  if (rc < 0) {
+    derr << __func__ << " invalid transport address: " << trid.traddr << dendl;
+    return -ENOENT;
+  }
   auto coremask_arg = g_conf().get_val<std::string>("bluestore_spdk_coremask");
   int m_core_arg = find_first_bitset(coremask_arg);
   // at least one core is needed for using spdk
@@ -569,8 +575,9 @@ int NVMEManager::try_get(const spdk_nvme_transport_id& trid, SharedDriverData **
 
   if (!dpdk_thread.joinable()) {
     dpdk_thread = std::thread(
-      [this, coremask_arg, m_core_arg, mem_size_arg]() {
-        static struct spdk_env_opts opts;
+      [this, coremask_arg, m_core_arg, mem_size_arg, pci_addr]() {
+        struct spdk_env_opts opts;
+        struct spdk_pci_addr addr = pci_addr;
         int r;
 
         spdk_env_opts_init(&opts);
@@ -578,6 +585,8 @@ int NVMEManager::try_get(const spdk_nvme_transport_id& trid, SharedDriverData **
         opts.core_mask = coremask_arg.c_str();
         opts.master_core = m_core_arg;
         opts.mem_size = mem_size_arg;
+        opts.pci_whitelist = &addr;
+        opts.num_pci_addr = 1;
         spdk_env_init(&opts);
         spdk_unaffinitize_thread();