From 125e02e0dd7ec669b7e6751e320082d53960d9ef Mon Sep 17 00:00:00 2001 From: Changcheng Liu Date: Wed, 15 Jul 2020 17:33:40 +0800 Subject: [PATCH] blk: split detect supported device and create driver Use detect_device_type to detect the devices supported by driver. Use create_with_type to create driver according to the device. Suggested-by: Kefu Chai Signed-off-by: Changcheng Liu --- src/blk/BlockDevice.cc | 91 ++++++++++++++++-------------------- src/blk/BlockDevice.h | 19 ++++++++ src/blk/pmem/PMEMDevice.cc | 14 ++++++ src/blk/pmem/PMEMDevice.h | 2 + src/blk/spdk/NVMEDevice.cc | 14 ++++++ src/blk/spdk/NVMEDevice.h | 2 + src/blk/zoned/HMSMRDevice.cc | 12 +++++ src/blk/zoned/HMSMRDevice.h | 1 + 8 files changed, 105 insertions(+), 50 deletions(-) diff --git a/src/blk/BlockDevice.cc b/src/blk/BlockDevice.cc index 029f0676de5..05e1dd24d89 100644 --- a/src/blk/BlockDevice.cc +++ b/src/blk/BlockDevice.cc @@ -29,14 +29,10 @@ #if defined(HAVE_BLUESTORE_PMEM) #include "pmem/PMEMDevice.h" -#include "libpmem.h" #endif #if defined(HAVE_LIBZBC) #include "zoned/HMSMRDevice.h" -extern "C" { -#include -} #endif #include "common/debug.h" @@ -90,67 +86,62 @@ void IOContext::release_running_aios() #endif } -BlockDevice *BlockDevice::create(CephContext* cct, const string& path, - aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv) +BlockDevice::block_device_t +BlockDevice::detect_device_type(const std::string& path) { - string type = "kernel"; - char buf[PATH_MAX + 1]; - int r = ::readlink(path.c_str(), buf, sizeof(buf) - 1); - if (r >= 0) { - buf[r] = '\0'; - char *bname = ::basename(buf); - if (strncmp(bname, SPDK_PREFIX, sizeof(SPDK_PREFIX)-1) == 0) - type = "ust-nvme"; - } - -#if defined(HAVE_BLUESTORE_PMEM) - if (type == "kernel") { - int is_pmem = 0; - size_t map_len = 0; - void *addr = pmem_map_file(path.c_str(), 0, PMEM_FILE_EXCL, O_RDONLY, &map_len, &is_pmem); - if (addr != NULL) { - if (is_pmem) - type = "pmem"; - else - dout(1) << path.c_str() << " isn't pmem file" << dendl; - pmem_unmap(addr, map_len); - } else { - dout(1) << "pmem_map_file:" << path.c_str() << " failed." << pmem_errormsg() << dendl; - } +#if defined(HAVE_SPDK) + if (NVMEDevice::support(path)) { + return block_device_t::spdk; } #endif - - dout(1) << __func__ << " path " << path << " type " << type << dendl; - #if defined(HAVE_BLUESTORE_PMEM) - if (type == "pmem") { - return new PMEMDevice(cct, cb, cbpriv); + if (PMEMDevice::support(path)) { + return block_device_t::pmem; } #endif -#if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO) -#if defined(HAVE_LIBZBC) - r = zbc_device_is_zoned(path.c_str(), false, nullptr); - if (r == 1) { - return new HMSMRDevice(cct, cb, cbpriv, d_cb, d_cbpriv); +#if (defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)) && defined(HAVE_LIZBC) + if (HMSMRDevice::support(path)) { + return block_device_t::hm_smr; } - ceph_assertf(r >= 0, "zbc_device_is_zoned(%s) failed: %d", path.c_str(), r); #endif - if (type == "kernel") { + + return block_device_t::aio; +} + +BlockDevice* BlockDevice::create_with_type(block_device_t device_type, + CephContext* cct, const std::string& path, aio_callback_t cb, + void *cbpriv, aio_callback_t d_cb, void *d_cbpriv) +{ + + switch (device_type) { +#if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO) + case block_device_t::aio: return new KernelDevice(cct, cb, cbpriv, d_cb, d_cbpriv); - } #endif -#ifndef WITH_SEASTAR #if defined(HAVE_SPDK) - if (type == "ust-nvme") { + case block_device_t::spdk: return new NVMEDevice(cct, cb, cbpriv); - } #endif +#if defined(HAVE_BLUESTORE_PMEM) + case block_device_t::pmem: + return new PMEMDevice(cct, cb, cbpriv); +#endif +#if (defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)) && defined(HAVE_LIZBC) + case block_device_t::hm_smr: + return new HMSMRDevice(cct, cb, cbpriv, d_cb, d_cbpriv); #endif + default: + ceph_abort_msg("unsupported device"); + return nullptr; + } +} - derr << __func__ << " unknown backend " << type << dendl; - - ceph_abort(); - return NULL; +BlockDevice *BlockDevice::create( + CephContext* cct, const string& path, aio_callback_t cb, + void *cbpriv, aio_callback_t d_cb, void *d_cbpriv) +{ + block_device_t device_type = detect_device_type(path); + return create_with_type(device_type, cct, path, cb, cbpriv, d_cb, d_cbpriv); } void BlockDevice::queue_reap_ioc(IOContext *ioc) diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h index b0929bad12d..a24411d0329 100644 --- a/src/blk/BlockDevice.h +++ b/src/blk/BlockDevice.h @@ -136,6 +136,25 @@ private: ceph::mutex ioc_reap_lock = ceph::make_mutex("BlockDevice::ioc_reap_lock"); std::vector ioc_reap_queue; std::atomic_int ioc_reap_count = {0}; + enum class block_device_t { + unknown, +#if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO) + aio, +#if defined(HAVE_LIBZBC) + hm_smr, +#endif +#endif +#if defined(HAVE_SPDK) + spdk, +#endif +#if defined(HAVE_BLUESTORE_PMEM) + pmem, +#endif + }; + static block_device_t detect_device_type(const std::string& path); + static BlockDevice *create_with_type(block_device_t device_type, + CephContext* cct, const std::string& path, aio_callback_t cb, + void *cbpriv, aio_callback_t d_cb, void *d_cbpriv); protected: uint64_t size = 0; diff --git a/src/blk/pmem/PMEMDevice.cc b/src/blk/pmem/PMEMDevice.cc index 3cdcacc047c..247ed0692ed 100644 --- a/src/blk/pmem/PMEMDevice.cc +++ b/src/blk/pmem/PMEMDevice.cc @@ -164,6 +164,20 @@ int PMEMDevice::collect_metadata(const string& prefix, map *pm) c return 0; } +bool PMEMDevice::support(const std::string &path) +{ + int is_pmem = 0; + size_t map_len = 0; + void *addr = pmem_map_file(path.c_str(), 0, PMEM_FILE_EXCL, O_RDONLY, &map_len, &is_pmem); + if (addr != NULL) { + if (is_pmem) { + return true; + } + pmem_unmap(addr, map_len); + } + return false; +} + int PMEMDevice::flush() { //Because all write is persist. So no need diff --git a/src/blk/pmem/PMEMDevice.h b/src/blk/pmem/PMEMDevice.h index 8b9bbfd25ab..a240d2a7b31 100644 --- a/src/blk/pmem/PMEMDevice.h +++ b/src/blk/pmem/PMEMDevice.h @@ -43,6 +43,8 @@ public: int collect_metadata(const std::string& prefix, map *pm) const override; + static bool support(const std::string& path); + int read(uint64_t off, uint64_t len, bufferlist *pbl, IOContext *ioc, bool buffered) override; diff --git a/src/blk/spdk/NVMEDevice.cc b/src/blk/spdk/NVMEDevice.cc index adf9e31abe7..7f3cde07994 100644 --- a/src/blk/spdk/NVMEDevice.cc +++ b/src/blk/spdk/NVMEDevice.cc @@ -698,6 +698,20 @@ NVMEDevice::NVMEDevice(CephContext* cct, aio_callback_t cb, void *cbpriv) { } +bool NVMEDevice::support(const std::string& path) +{ + char buf[PATH_MAX + 1]; + int r = ::readlink(path.c_str(), buf, sizeof(buf) - 1); + if (r >= 0) { + buf[r] = '\0'; + char *bname = ::basename(buf); + if (strncmp(bname, SPDK_PREFIX, sizeof(SPDK_PREFIX)-1) == 0) { + return true; + } + } + return false; +} + int NVMEDevice::open(const string& p) { dout(1) << __func__ << " path " << p << dendl; diff --git a/src/blk/spdk/NVMEDevice.h b/src/blk/spdk/NVMEDevice.h index 9ad56854908..352856093d8 100644 --- a/src/blk/spdk/NVMEDevice.h +++ b/src/blk/spdk/NVMEDevice.h @@ -55,6 +55,8 @@ class NVMEDevice : public BlockDevice { bool supported_bdev_label() override { return false; } + static bool support(const std::string& path); + void aio_submit(IOContext *ioc) override; int read(uint64_t off, uint64_t len, bufferlist *pbl, diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc index 3ae72c35f83..a8b574f0562 100644 --- a/src/blk/zoned/HMSMRDevice.cc +++ b/src/blk/zoned/HMSMRDevice.cc @@ -76,6 +76,18 @@ HMSMRDevice::HMSMRDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_ } } +bool HMSMRDevice::support(const std::string& path) +{ + int r = zbc_device_is_zoned(path.c_str(), false, nullptr); + if (r == 1) { + return true; + } else if (r < 0) { + derr << __func__ << " zbc_device_is_zoned(" << path << ") failed: " + << cpp_strerror(r) << dendl; + } + return false; +} + int HMSMRDevice::_lock() { dout(10) << __func__ << " " << fd_directs[WRITE_LIFE_NOT_SET] << dendl; diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h index 5ec8ee7d0ff..30941f2f9c6 100644 --- a/src/blk/zoned/HMSMRDevice.h +++ b/src/blk/zoned/HMSMRDevice.h @@ -117,6 +117,7 @@ class HMSMRDevice final : public BlockDevice { public: HMSMRDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv); + static bool support(const std::string& path); void aio_submit(IOContext *ioc) final; void discard_drain() final; -- 2.39.5