From bc7c6b3fc0f3d0f67d4eb6f9b1b4cc6433731a94 Mon Sep 17 00:00:00 2001 From: optimistyzy Date: Fri, 14 Apr 2017 13:38:40 +0800 Subject: [PATCH] bluestore/NVMEDEVICE: update SPDK to version 17.03 Do some minor changes: 1 Restrict the total DPDK memory used by an osd instance. change the name from bluestore_spdk_socket_mem to bluestore_spdk_mem. 2 use spdk_env_init instead of rte_eal_init. The reason is that SPDK lib invokes rte_eal_init which reduces the initilization paramter conversion and check, also spdk 17.03 invokes spdk_vtophys_register_dpdk_mem() (which is an internal function) in spdk_env_init, and this func must be called. Signed-off-by: optimistyzy --- src/common/config_opts.h | 5 ++-- src/os/bluestore/NVMEDevice.cc | 51 +++++++++++----------------------- src/spdk | 2 +- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index c742ae6fa20..2dc3700b997 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1032,8 +1032,9 @@ OPTION(bluestore_bluefs_reclaim_ratio, OPT_FLOAT, .20) // how much to reclaim at // Example: // bluestore_block_path = spdk:55cd2e404bd73932 // If you want to run multiple SPDK instances per node, you must specify the -// amount of memory per socket each instance will use. -OPTION(bluestore_spdk_socket_mem, OPT_STR, "512,512") +// amount of dpdk memory size in MB each instance will use, to make sure each +// instance uses its own dpdk memory +OPTION(bluestore_spdk_mem, OPT_U32, 512) // A hexadecimal bit mask of the cores to run on. Note the core numbering can change between platforms and should be determined beforehand. OPTION(bluestore_spdk_coremask, OPT_STR, "0x3") // Specify the maximal I/Os to be batched completed while checking queue pair completions. diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc index 6b08bf707f2..2a6a5bfc3b3 100644 --- a/src/os/bluestore/NVMEDevice.cc +++ b/src/os/bluestore/NVMEDevice.cc @@ -707,20 +707,9 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) int r = 0; unsigned long long core_value; uint32_t core_num = 0; - int m_core = -1; - - string master_core = "--master-lcore="; - string prefix = "--file-prefix="; - string sock_mem = "--socket-mem="; - string coremask = "-c "; - stringstream master_core_ss; - prefix += sn_tag; - sock_mem += g_conf->bluestore_spdk_socket_mem; - coremask += g_conf->bluestore_spdk_coremask; - char *prefix_arg = (char *)prefix.c_str(); - char *sock_mem_arg = (char *)sock_mem.c_str(); - char *coremask_arg = (char *)coremask.c_str(); - char *master_core_arg; + int m_core_arg = -1; + uint32_t mem_size_arg = g_conf->bluestore_spdk_mem; + char *coremask_arg = (char *)g_conf->bluestore_spdk_coremask.c_str(); if (sn_tag.empty()) { r = -ENOENT; @@ -728,14 +717,14 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) return r; } - core_value = strtoll(g_conf->bluestore_spdk_coremask.c_str(), NULL, 16); + core_value = strtoll(coremask_arg, NULL, 16); for (uint32_t i = 0; i < sizeof(long long) * 8; i++) { bool tmp = (core_value >> i) & 0x1; if (tmp) { core_num += 1; // select the least signficant bit as the master core - if(m_core < 0) { - m_core = i; + if(m_core_arg < 0) { + m_core_arg = i; } } } @@ -747,9 +736,6 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) << cpp_strerror(r) << dendl; return r; } - master_core_ss << m_core; - master_core += master_core_ss.str(); - master_core_arg = (char *)master_core.c_str(); for (auto &&it : shared_driver_datas) { if (it->is_equal(sn_tag)) { @@ -761,21 +747,16 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) if (!init) { init = true; dpdk_thread = std::thread( - [this, prefix_arg, sock_mem_arg, coremask_arg, master_core_arg]() { - static const char *ealargs[] = { - "ceph-osd", - coremask_arg, /* This must be the second parameter. It is overwritten by index in main(). */ - "-n 4", - sock_mem_arg, - prefix_arg, - master_core_arg - }; - - int r = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), (char **)(void *)(uintptr_t)ealargs); - if (r < 0) { - derr << __func__ << " failed to do rte_eal_init" << dendl; - ceph_abort(); - } + [this, coremask_arg, m_core_arg, mem_size_arg]() { + static struct spdk_env_opts opts; + int r; + + spdk_env_opts_init(&opts); + opts.name = "ceph-osd"; + opts.core_mask = coremask_arg; + opts.dpdk_master_core = m_core_arg; + opts.dpdk_mem_size = mem_size_arg; + spdk_env_init(&opts); spdk_nvme_retry_count = g_ceph_context->_conf->bdev_nvme_retry_count; if (spdk_nvme_retry_count < 0) diff --git a/src/spdk b/src/spdk index df46c41a4ca..5742e9b9e78 160000 --- a/src/spdk +++ b/src/spdk @@ -1 +1 @@ -Subproject commit df46c41a4ca2edabd642a73b25e54dcb173cf976 +Subproject commit 5742e9b9e782b9666e10c9224389e4d015c3cdee -- 2.39.5