From: Kefu Chai Date: Tue, 18 Sep 2018 12:23:23 +0000 (+0800) Subject: bluestore/NVMe: use PCIe selector as the path name X-Git-Tag: v14.0.1~115^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=78116ce147be0b604e1efede2198b5fe9d509d84;p=ceph.git bluestore/NVMe: use PCIe selector as the path name as the latest SPDK deprecated and removed spdk_pci_get_device(), we cannot use the PCI device's serial number as its identifier and the filename of the file representing the device. in this change, we are using the PCI device's selector instead, it is also used by SPDK as part of the transport id. Tested-by: Tone Zhang Signed-off-by: Kefu Chai --- diff --git a/doc/rados/configuration/bluestore-config-ref.rst b/doc/rados/configuration/bluestore-config-ref.rst index b15ed2bcedf8..7f9f35005ce0 100644 --- a/doc/rados/configuration/bluestore-config-ref.rst +++ b/doc/rados/configuration/bluestore-config-ref.rst @@ -392,16 +392,21 @@ script as root:: $ sudo src/spdk/scripts/setup.sh -Then you need to specify NVMe serial number here with "spdk:" prefix for +Then you need to specify NVMe device's device selector here with "spdk:" prefix for ``bluestore_block_path``. -For example, users can find the serial number with:: +For example, users can find the device selector of an Intel PCIe SSD with:: - $ lspci -vvv -d 8086:0953 | grep "Device Serial Number" + $ lspci -mm -n -D -d 8086:0953 + +The device selector always has the form of ``DDDD:BB:DD.FF`` or ``DDDD.BB.DD.FF``. and then set:: - bluestore block path = spdk:... + bluestore block path = spdk:0000:01:00.0 + +Where ``0000:01:00.0`` is the device selector found in the output of ``lspci`` +command above. If you want to run multiple SPDK instances per node, you must specify the amount of dpdk memory size in MB each instance will use, to make sure each diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index fa1aa01b4238..2ff1d2ae49ba 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5494,9 +5494,15 @@ int BlueStore::_setup_block_symlink_or_file( << cpp_strerror(r) << dendl; return r; } - string serial_number = epath.substr(strlen(SPDK_PREFIX)); - r = ::write(fd, serial_number.c_str(), serial_number.size()); - ceph_assert(r == (int)serial_number.size()); + // write the Transport ID of the NVMe device + // a transport id looks like: "trtype:PCIe traddr:0000:02:00.0" + // where "0000:02:00.0" is the selector of a PCI device, see + // the first column of "lspci -mm -n -D" + string trid{"trtype:PCIe "}; + trid += "traddr:"; + trid += epath.substr(strlen(SPDK_PREFIX)); + r = ::write(fd, trid.c_str(), trid.size()); + ceph_assert(r == static_cast(trid.size())); dout(1) << __func__ << " created " << name << " symlink to " << epath << dendl; VOID_TEMP_FAILURE_RETRY(::close(fd)); diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc index b5288ea42e8d..cce6faece30f 100644 --- a/src/os/bluestore/NVMEDevice.cc +++ b/src/os/bluestore/NVMEDevice.cc @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -82,7 +83,7 @@ struct Task; class SharedDriverData { unsigned id; - std::string sn; + spdk_nvme_transport_id trid; spdk_nvme_ctrlr *ctrlr; spdk_nvme_ns *ns; uint64_t block_size = 0; @@ -92,18 +93,20 @@ class SharedDriverData { public: std::vector registered_devices; friend class SharedDriverQueueData; - SharedDriverData(unsigned _id, const std::string &sn_tag, - spdk_nvme_ctrlr *c, spdk_nvme_ns *ns) - : id(_id), - sn(sn_tag), + SharedDriverData(unsigned id_, const spdk_nvme_transport_id& trid_, + spdk_nvme_ctrlr *c, spdk_nvme_ns *ns_) + : id(id_), + trid(trid_), ctrlr(c), - ns(ns) { + ns(ns_) { sector_size = spdk_nvme_ns_get_sector_size(ns); block_size = std::max(CEPH_PAGE_SIZE, sector_size); size = ((uint64_t)sector_size) * spdk_nvme_ns_get_num_sectors(ns); } - bool is_equal(const string &tag) const { return sn == tag; } + bool is_equal(const spdk_nvme_transport_id& trid2) const { + return spdk_nvme_transport_id_compare(&trid, &trid2) == 0; + } ~SharedDriverData() { } @@ -471,7 +474,7 @@ void SharedDriverQueueData::_aio_handle(Task *t, IOContext *ioc) class NVMEManager { public: struct ProbeContext { - string sn_tag; + spdk_nvme_transport_id trid; NVMEManager *manager; SharedDriverData *driver; bool done; @@ -489,9 +492,8 @@ class NVMEManager { public: NVMEManager() : lock("NVMEDevice::NVMEManager::lock") {} - int try_get(const string &sn_tag, SharedDriverData **driver); - void register_ctrlr(const string &sn_tag, spdk_nvme_ctrlr *c, struct spdk_pci_device *pci_dev, - SharedDriverData **driver) { + int try_get(const spdk_nvme_transport_id& trid, SharedDriverData **driver); + void register_ctrlr(const spdk_nvme_transport_id& trid, spdk_nvme_ctrlr *c, SharedDriverData **driver) { ceph_assert(lock.is_locked()); spdk_nvme_ns *ns; int num_ns = spdk_nvme_ctrlr_get_num_ns(c); @@ -504,13 +506,12 @@ class NVMEManager { derr << __func__ << " failed to get namespace at 1" << dendl; ceph_abort(); } - dout(1) << __func__ << " successfully attach nvme device at" << spdk_pci_device_get_bus(pci_dev) - << ":" << spdk_pci_device_get_dev(pci_dev) << ":" << spdk_pci_device_get_func(pci_dev) << dendl; + dout(1) << __func__ << " successfully attach nvme device at" << trid.traddr << dendl; // only support one device per osd now! ceph_assert(shared_driver_datas.empty()); // index 0 is occurred by master thread - shared_driver_datas.push_back(new SharedDriverData(shared_driver_datas.size()+1, sn_tag, c, ns)); + shared_driver_datas.push_back(new SharedDriverData(shared_driver_datas.size()+1, trid, c, ns)); *driver = shared_driver_datas.back(); } }; @@ -520,40 +521,17 @@ static NVMEManager manager; static bool probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, struct spdk_nvme_ctrlr_opts *opts) { NVMEManager::ProbeContext *ctx = static_cast(cb_ctx); - char serial_number[128]; - struct spdk_pci_addr pci_addr; - struct spdk_pci_device *pci_dev = NULL; - int result = 0; if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) { dout(0) << __func__ << " only probe local nvme device" << dendl; return false; } - result = spdk_pci_addr_parse(&pci_addr, trid->traddr); - if (result) { - dout(0) << __func__ << " failed to get pci address from %s, " << trid->traddr << " return value is: %d" << result << dendl; - return false; - } - - pci_dev = spdk_pci_get_device(&pci_addr); - if (!pci_dev) { - dout(0) << __func__ << " failed to get pci device" << dendl; - return false; - } - - dout(0) << __func__ << " found device at bus: " << spdk_pci_device_get_bus(pci_dev) - << ":" << spdk_pci_device_get_dev(pci_dev) << ":" - << spdk_pci_device_get_func(pci_dev) << " vendor:0x" << spdk_pci_device_get_vendor_id(pci_dev) << " device:0x" << spdk_pci_device_get_device_id(pci_dev) - << dendl; - result = spdk_pci_device_get_serial_number(pci_dev, serial_number, 128); - if (result < 0) { - dout(10) << __func__ << " failed to get serial number from %p" << pci_dev << dendl; - return false; - } - - if (ctx->sn_tag.compare(string(serial_number, 16))) { - dout(0) << __func__ << " device serial number (" << ctx->sn_tag << ") not match " << serial_number << dendl; + dout(0) << __func__ << " found device at: " + << "trtype=" << spdk_nvme_transport_id_trtype_str(trid->trtype) << ", " + << "traddr=" << trid->traddr << dendl; + if (spdk_nvme_transport_id_compare(&ctx->trid, trid)) { + dout(0) << __func__ << " device traddr (" << ctx->trid.traddr << ") not match " << trid->traddr << dendl; return false; } @@ -563,31 +541,15 @@ static bool probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, st static void attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) { - struct spdk_pci_addr pci_addr; - struct spdk_pci_device *pci_dev = NULL; - - spdk_pci_addr_parse(&pci_addr, trid->traddr); - - pci_dev = spdk_pci_get_device(&pci_addr); - if (!pci_dev) { - dout(0) << __func__ << " failed to get pci device" << dendl; - ceph_assert(pci_dev); - } - - NVMEManager::ProbeContext *ctx = static_cast(cb_ctx); - ctx->manager->register_ctrlr(ctx->sn_tag, ctrlr, pci_dev, &ctx->driver); + auto ctx = static_cast(cb_ctx); + ctx->manager->register_ctrlr(ctx->trid, ctrlr, &ctx->driver); } -int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) +int NVMEManager::try_get(const spdk_nvme_transport_id& trid, SharedDriverData **driver) { Mutex::Locker l(lock); - if (sn_tag.empty()) { - derr << __func__ << " empty serial number" << dendl; - return -ENOENT; - } - for (auto &&it : shared_driver_datas) { - if (it->is_equal(sn_tag)) { + if (it->is_equal(trid)) { *driver = it; return 0; } @@ -654,7 +616,7 @@ int NVMEManager::try_get(const string &sn_tag, SharedDriverData **driver) dpdk_thread.detach(); } - ProbeContext ctx = {sn_tag, this, nullptr, false}; + ProbeContext ctx{trid, this, nullptr, false}; { std::unique_lock l(probe_queue_lock); probe_queue.push_back(&ctx); @@ -736,46 +698,30 @@ NVMEDevice::NVMEDevice(CephContext* cct, aio_callback_t cb, void *cbpriv) int NVMEDevice::open(const string& p) { - int r = 0; dout(1) << __func__ << " path " << p << dendl; - string serial_number; - int fd = ::open(p.c_str(), O_RDONLY | O_CLOEXEC); - if (fd < 0) { - r = -errno; - derr << __func__ << " unable to open " << p << ": " << cpp_strerror(r) - << dendl; - return r; + std::ifstream ifs(p); + if (!ifs) { + derr << __func__ << " unable to open " << p << dendl; + return -1; } - char buf[100]; - r = ::read(fd, buf, sizeof(buf)); - VOID_TEMP_FAILURE_RETRY(::close(fd)); - fd = -1; // defensive - if (r <= 0) { - if (r == 0) { - r = -EINVAL; - } else { - r = -errno; - } - derr << __func__ << " unable to read " << p << ": " << cpp_strerror(r) << dendl; + string val; + std::getline(ifs, val); + spdk_nvme_transport_id trid; + if (int r = spdk_nvme_transport_id_parse(&trid, val.c_str()); r) { + derr << __func__ << " unable to read " << p << ": " << cpp_strerror(r) + << dendl; return r; } - /* scan buf from the beginning with isxdigit. */ - int i = 0; - while (i < r && isxdigit(buf[i])) { - i++; - } - serial_number = string(buf, i); - r = manager.try_get(serial_number, &driver); - if (r < 0) { - derr << __func__ << " failed to get nvme device with sn " << serial_number << dendl; + if (int r = manager.try_get(trid, &driver); r < 0) { + derr << __func__ << " failed to get nvme device with transport address " << trid.traddr << dendl; return r; } driver->register_device(this); block_size = driver->get_block_size(); size = driver->get_size(); - name = serial_number; + name = trid.traddr; //nvme is non-rotational device. rotational = false; diff --git a/src/vstart.sh b/src/vstart.sh index 6cfd2478278f..50f11cc2e45e 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -435,8 +435,8 @@ wconf() { fi } -get_nvme_serial_number() { - sudo lspci -vvv -d $pci_id | grep 'Device Serial Number' | awk '{print $NF}' | sed 's/-//g' +get_pci_selector() { + lspci -mm -n -D -d $pci_id | cut -d' ' -f1 } prepare_conf() { @@ -496,7 +496,7 @@ EOF fi if [ "$objectstore" == "bluestore" ]; then if [ "$spdk_enabled" -eq 1 ]; then - if [ "$(get_nvme_serial_number)" == "" ]; then + if [ "$(get_pci_selector)" == "" ]; then echo "Not find the specified NVME device, please check." exit fi @@ -507,7 +507,7 @@ EOF bluestore_block_wal_size = 0 bluestore_block_wal_create = false bluestore_spdk_mem = 2048 - bluestore_block_path = spdk:$(get_nvme_serial_number)" + bluestore_block_path = spdk:$(get_pci_selector)" else BLUESTORE_OPTS=" bluestore block db path = $CEPH_DEV_DIR/osd\$id/block.db.file bluestore block db size = 67108864