bluestore/NVMEDEVICE: update SPDK to version 17.03

Do some minor changes:

1 Restrict the total DPDK memory used by an osd instance.
change the name from bluestore_spdk_socket_mem to
bluestore_spdk_mem.

2 use spdk_env_init instead of rte_eal_init. The reason is that
SPDK lib invokes rte_eal_init which reduces the initilization
paramter conversion and check, also spdk 17.03 invokes
spdk_vtophys_register_dpdk_mem() (which is an internal function)
in spdk_env_init, and this func must be called.

Signed-off-by: optimistyzy <optimistyzy@gmail.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index c742ae6fa205388827af3438fd2900ba66773cd4..2dc3700b997d4c6808e3bfb6e56a2d566190b923 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1032,8 +1032,9 @@ OPTION(bluestore_bluefs_reclaim_ratio, OPT_FLOAT, .20) // how much to reclaim at
 // Example:
 // bluestore_block_path = spdk:55cd2e404bd73932
 // If you want to run multiple SPDK instances per node, you must specify the
-// amount of memory per socket each instance will use.
-OPTION(bluestore_spdk_socket_mem, OPT_STR, "512,512")
+// amount of dpdk memory size in MB each instance will use, to make sure each
+// instance uses its own dpdk memory
+OPTION(bluestore_spdk_mem, OPT_U32, 512)
 // A hexadecimal bit mask of the cores to run on. Note the core numbering can change between platforms and should be determined beforehand.
 OPTION(bluestore_spdk_coremask, OPT_STR, "0x3")
 // Specify the maximal I/Os to be batched completed while checking queue pair completions.

diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc
index 6b08bf707f2322f642147cbf188e410fe330512e..2a6a5bfc3b36e9f6ced79940838eafe89c8a565a 100644 (file)
--- a/src/os/bluestore/NVMEDevice.cc
+++ b/src/os/bluestore/NVMEDevice.cc
@@ -707,20 +707,9 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver)
   int r = 0;
   unsigned long long core_value;
   uint32_t core_num = 0;
-  int m_core = -1;
-
-  string master_core = "--master-lcore=";
-  string prefix = "--file-prefix=";
-  string sock_mem = "--socket-mem=";
-  string coremask = "-c ";
-  stringstream master_core_ss;
-  prefix += sn_tag;
-  sock_mem += g_conf->bluestore_spdk_socket_mem;
-  coremask += g_conf->bluestore_spdk_coremask;
-  char *prefix_arg = (char *)prefix.c_str();
-  char *sock_mem_arg = (char *)sock_mem.c_str();
-  char *coremask_arg = (char *)coremask.c_str();
-  char *master_core_arg;
+  int m_core_arg = -1;
+  uint32_t mem_size_arg = g_conf->bluestore_spdk_mem;
+  char *coremask_arg = (char *)g_conf->bluestore_spdk_coremask.c_str();
 
   if (sn_tag.empty()) {
     r = -ENOENT;
@@ -728,14 +717,14 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver)
     return r;
   }
 
-  core_value = strtoll(g_conf->bluestore_spdk_coremask.c_str(), NULL, 16);
+  core_value = strtoll(coremask_arg, NULL, 16);
   for (uint32_t i = 0; i < sizeof(long long) * 8; i++) {
     bool tmp = (core_value >> i) & 0x1;
     if (tmp) {
       core_num += 1;
       // select the least signficant bit as the master core
-      if(m_core < 0) {
-        m_core = i;
+      if(m_core_arg < 0) {
+        m_core_arg = i;
       }
     }
   }
@@ -747,9 +736,6 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver)
          << cpp_strerror(r) << dendl;
     return r;
   }
-  master_core_ss << m_core;
-  master_core += master_core_ss.str();
-  master_core_arg = (char *)master_core.c_str();
 
   for (auto &&it : shared_driver_datas) {
     if (it->is_equal(sn_tag)) {
@@ -761,21 +747,16 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver)
   if (!init) {
     init = true;
     dpdk_thread = std::thread(
-      [this, prefix_arg, sock_mem_arg, coremask_arg, master_core_arg]() {
-        static const char *ealargs[] = {
-            "ceph-osd",
-            coremask_arg, /* This must be the second parameter. It is overwritten by index in main(). */
-            "-n 4",
-           sock_mem_arg,
-           prefix_arg,
-           master_core_arg
-        };
-
-        int r = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), (char **)(void *)(uintptr_t)ealargs);
-        if (r < 0) {
-          derr << __func__ << " failed to do rte_eal_init" << dendl;
-          ceph_abort();
-        }
+      [this, coremask_arg, m_core_arg, mem_size_arg]() {
+        static struct spdk_env_opts opts;
+        int r;
+
+        spdk_env_opts_init(&opts);
+        opts.name = "ceph-osd";
+        opts.core_mask = coremask_arg;
+        opts.dpdk_master_core = m_core_arg;
+        opts.dpdk_mem_size = mem_size_arg;
+        spdk_env_init(&opts);
 
         spdk_nvme_retry_count = g_ceph_context->_conf->bdev_nvme_retry_count;
         if (spdk_nvme_retry_count < 0)

diff --git a/src/spdk b/src/spdk
index df46c41a4ca2edabd642a73b25e54dcb173cf976..5742e9b9e782b9666e10c9224389e4d015c3cdee 160000 (submodule)
--- a/src/spdk
+++ b/src/spdk
@@ -1 +1 @@
-Subproject commit df46c41a4ca2edabd642a73b25e54dcb173cf976
+Subproject commit 5742e9b9e782b9666e10c9224389e4d015c3cdee