From: Haomai Wang Date: Tue, 5 Jan 2016 09:57:44 +0000 (+0800) Subject: NVMEDevice: SPDK only permit submit/poll within one thread X-Git-Tag: v10.0.4~81^2~36 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2ad7c18222418efaa807e2feb5cd17ac5a39d931;p=ceph.git NVMEDevice: SPDK only permit submit/poll within one thread Signed-off-by: Haomai Wang --- diff --git a/src/os/bluestore/BlockDevice.h b/src/os/bluestore/BlockDevice.h index 73d7cac74580..03fa9707ab41 100644 --- a/src/os/bluestore/BlockDevice.h +++ b/src/os/bluestore/BlockDevice.h @@ -24,6 +24,7 @@ struct IOContext { void *priv; #ifdef HAVE_SPDK + bool done = false; void *backend; #endif diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc index 6f3eff07dd0d..82f1ad7acc0c 100644 --- a/src/os/bluestore/NVMEDevice.cc +++ b/src/os/bluestore/NVMEDevice.cc @@ -31,6 +31,7 @@ #include "include/compat.h" #include "common/errno.h" #include "common/debug.h" +#include "common/Initialize.h" #include "NVMEDevice.h" @@ -60,6 +61,7 @@ static void io_complete(void *ctx, const struct nvme_completion *completion) { IOContext *ioc = (IOContext*)ctx; NVMEDevice *device = (NVMEDevice*)ioc->backend; + ioc->done = true; if (ioc->priv) { device->aio_callback(device->aio_callback_priv, ioc->priv); } @@ -72,8 +74,6 @@ static void io_complete(void *ctx, const struct nvme_completion *completion) { NVMEDevice::NVMEDevice(aio_callback_t cb, void *cbpriv) : ctrlr(nullptr), ns(nullptr), - aio_stop(false), - aio_thread(this), aio_callback(cb), aio_callback_priv(cbpriv) { @@ -87,23 +87,28 @@ static char *ealargs[] = { "-n 4", }; -int NVMEDevice::open(string p) +void NVMEDevice::init() { - int r = 0; - dout(1) << __func__ << " path " << p << dendl; - - pci_device *pci_dev; - - r = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), (char **)(void *)(uintptr_t)ealargs); + int r = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), (char **)(void *)(uintptr_t)ealargs); if (r < 0) { derr << __func__ << " init dpdk failed" << dendl; - return r; + assert(0); } if (request_mempool == NULL) { derr << __func__ << " could not initialize request mempool" << dendl; - return -1; + assert(0); } +} + +int NVMEDevice::open(string p) +{ + static Initialize _(NVMEDevice::init); + + int r = 0; + dout(1) << __func__ << " path " << p << dendl; + + pci_device *pci_dev; pci_system_init(); @@ -145,13 +150,14 @@ int NVMEDevice::open(string p) continue; } + name = pci_device_get_device_name(pci_dev) ? pci_device_get_device_name(pci_dev) : "Unknown"; if (pci_device_has_kernel_driver(pci_dev)) { if (!pci_device_has_uio_driver(pci_dev)) { /*NVMe kernel driver case*/ if (g_conf->bdev_nvme_unbind_from_kernel) { r = pci_device_switch_to_uio_driver(pci_dev); if (r < 0) { - derr << __func__ << " device " << pci_device_get_device_name(pci_dev) << " " << pci_dev->bus + derr << __func__ << " device " << name << " " << pci_dev->bus << ":" << pci_dev->dev << ":" << pci_dev->func << " switch to uio driver failed" << dendl; return r; @@ -165,7 +171,7 @@ int NVMEDevice::open(string p) } else { r = pci_device_bind_uio_driver(pci_dev, PCI_UIO_DRIVER); if (r < 0) { - derr << __func__ << " device " << pci_device_get_device_name(pci_dev) << " " << pci_dev->bus + derr << __func__ << " device " << name << " " << pci_dev->bus << ":" << pci_dev->dev << ":" << pci_dev->func << " bind to uio driver failed" << dendl; return r; @@ -175,7 +181,7 @@ int NVMEDevice::open(string p) /* Claim the device in case conflict with other ids process */ r = pci_device_claim(pci_dev); if (r < 0) { - derr << __func__ << " device " << pci_device_get_device_name(pci_dev) << " " << pci_dev->bus + derr << __func__ << " device " << name << " " << pci_dev->bus << ":" << pci_dev->dev << ":" << pci_dev->func << " claim failed" << dendl; return r; @@ -217,10 +223,6 @@ int NVMEDevice::open(string p) << " block_size " << block_size << " (" << pretty_si_t(block_size) << "B)" << dendl; - r = nvme_register_io_thread(); - assert(r == 0); - r = _aio_start(); - assert(r == 0); name = pci_device_get_device_name(pci_dev); return 0; } @@ -229,7 +231,6 @@ void NVMEDevice::close() { dout(1) << __func__ << dendl; nvme_unregister_io_thread(); - _aio_stop(); name.clear(); } @@ -239,31 +240,12 @@ int NVMEDevice::flush() return 0; } -int NVMEDevice::_aio_start() -{ - dout(10) << __func__ << dendl; - aio_thread.create(); - return 0; -} - -void NVMEDevice::_aio_stop() +void NVMEDevice::aio_submit(IOContext *ioc) { - dout(10) << __func__ << dendl; - aio_stop = true; - aio_thread.join(); - aio_stop = false; -} - -void NVMEDevice::_aio_thread() -{ - dout(10) << __func__ << " start" << dendl; - while (!aio_stop) { - dout(40) << __func__ << " polling" << dendl; - - nvme_ctrlr_process_io_completions(ctrlr, 0); - + while (!ioc->done) { + nvme_ctrlr_process_io_completions(ctrlr, 0); + usleep(50); } - dout(10) << __func__ << " end" << dendl; } int NVMEDevice::aio_write( @@ -342,6 +324,10 @@ int NVMEDevice::read(uint64_t off, uint64_t len, bufferlist *pbl, derr << __func__ << " failed to read" << dendl; return r; } + while (!ioc->done) { + nvme_ctrlr_process_io_completions(ctrlr, 0); + usleep(50); + } pbl->clear(); pbl->push_back(p); diff --git a/src/os/bluestore/NVMEDevice.h b/src/os/bluestore/NVMEDevice.h index 2d3871b4664f..2e14fbab2d9d 100644 --- a/src/os/bluestore/NVMEDevice.h +++ b/src/os/bluestore/NVMEDevice.h @@ -29,12 +29,12 @@ extern "C" { #include "spdk/pci.h" #include "spdk/nvme.h" -#include "BlockDevice.h" - #ifdef __cplusplus } #endif +#include "BlockDevice.h" + class NVMEDevice : public BlockDevice { /** * points to pinned, physically contiguous memory region; @@ -48,21 +48,9 @@ class NVMEDevice : public BlockDevice { uint64_t size; uint64_t block_size; - bool aio_stop; bufferptr zeros; - struct AioCompletionThread : public Thread { - NVMEDevice *bdev; - AioCompletionThread(NVMEDevice *b) : bdev(b) {} - void *entry() { - bdev->_aio_thread(); - return NULL; - } - } aio_thread; - - void _aio_thread(); - int _aio_start(); - void _aio_stop(); + static void init(); public: aio_callback_t aio_callback; @@ -70,7 +58,7 @@ class NVMEDevice : public BlockDevice { NVMEDevice(aio_callback_t cb, void *cbpriv); - void aio_submit(IOContext *ioc) override {} + void aio_submit(IOContext *ioc) override; uint64_t get_size() const override { return size;