From: Ziye Yang <ziye.yang@intel.com>
Date: Tue, 7 Jul 2020 17:19:36 +0000 (+0800)
Subject: NVMEDevice: Fix the issue of multiple OSDs when using SPDK
X-Git-Tag: v16.1.0~1759^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F35968%2Fhead;p=ceph.git

NVMEDevice: Fix the issue of multiple OSDs when using SPDK

For each OSD when using SPDK, it should be the type of primary
process. So when there are multiple NVMe SSDs, each OSD will use
one. So we do not want an OSD process claim all the NVMe SSDs,
so we use the allowed list to restrict the probe.

And with this patch, we can support multiple OSDs while using
SPDK.

Signed-off-by: Ziye Yang <ziye.yang@intel.com>
---

diff --git a/src/blk/spdk/NVMEDevice.cc b/src/blk/spdk/NVMEDevice.cc
index 69ae45acf121..adf9e31abe71 100644
--- a/src/blk/spdk/NVMEDevice.cc
+++ b/src/blk/spdk/NVMEDevice.cc
@@ -555,6 +555,12 @@ int NVMEManager::try_get(const spdk_nvme_transport_id& trid, SharedDriverData **
     }
   }
 
+  struct spdk_pci_addr pci_addr;
+  int rc = spdk_pci_addr_parse(&pci_addr, trid.traddr);
+  if (rc < 0) {
+    derr << __func__ << " invalid transport address: " << trid.traddr << dendl;
+    return -ENOENT;
+  }
   auto coremask_arg = g_conf().get_val<std::string>("bluestore_spdk_coremask");
   int m_core_arg = find_first_bitset(coremask_arg);
   // at least one core is needed for using spdk
@@ -569,8 +575,9 @@ int NVMEManager::try_get(const spdk_nvme_transport_id& trid, SharedDriverData **
 
   if (!dpdk_thread.joinable()) {
     dpdk_thread = std::thread(
-      [this, coremask_arg, m_core_arg, mem_size_arg]() {
-        static struct spdk_env_opts opts;
+      [this, coremask_arg, m_core_arg, mem_size_arg, pci_addr]() {
+        struct spdk_env_opts opts;
+        struct spdk_pci_addr addr = pci_addr;
         int r;
 
         spdk_env_opts_init(&opts);
@@ -578,6 +585,8 @@ int NVMEManager::try_get(const spdk_nvme_transport_id& trid, SharedDriverData **
         opts.core_mask = coremask_arg.c_str();
         opts.master_core = m_core_arg;
         opts.mem_size = mem_size_arg;
+        opts.pci_whitelist = &addr;
+        opts.num_pci_addr = 1;
         spdk_env_init(&opts);
         spdk_unaffinitize_thread();