NVMEDevice: fix unknown serial number problem

Signed-off-by: Haomai Wang <haomai@xsky.com>

diff --git a/configure.ac b/configure.ac
index e6e541e839c9451b9c98613d8552a8b043d3c6e9..3218f75c365693809cbd87e17525356f23c91e14 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -721,6 +721,24 @@ if test "x$enable_xio" = x"yes"; then
    AC_SUBST(XIO_LIBS)
 fi
 
+#
+# DPDK
+#
+
+AC_ARG_WITH(dpdk,
+    [AC_HELP_STRING([--with-dpdk], [build Ceph DPDK Support])], [], [with_dpdk=no])
+
+AM_CONDITIONAL(WITH_DPDK, [test "x$with_dpdk" != "xno"])
+
+if test "x$with_dpdk" != x"no"; then
+   CPPFLAGS="$CPPFLAGS -I$with_dpdk/include"
+   LDFLAGS="$LDFLAGS -I$with_dpdk/lib"
+   AC_CHECK_HEADER([rte_config.h], [], AC_MSG_ERROR([Cannot find header 'rte_config.h'.]))
+   AC_CHECK_LIB([rte_eal], [rte_eal_init], [], AC_MSG_FAILURE([DPDK rte_eal_init not found]))
+
+   AC_DEFINE([HAVE_DPDK], [1], [DPDK conditional compilation])
+fi
+
 #
 # SPDK
 #
@@ -731,6 +749,7 @@ AM_CONDITIONAL(WITH_SPDK, [test "x$with_spdk" != "xno"])
 
 if test "x$with_spdk" != x"no"; then
    CPPFLAGS="$CPPFLAGS -I$with_spdk/include"
+   LDFLAGS="$LDFLAGS -I$with_dpdk/lib"
    AC_CHECK_HEADER([spdk/nvme.h], [], AC_MSG_ERROR([Cannot find header 'spdk/nvme.h'.]))
    AC_CHECK_LIB([spdk_nvme], [nvme_attach], [], AC_MSG_FAILURE([SPDK nvme_attach not found]))
 

diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc
index aec24475b9debe014596e3f72a378279ec9d2a68..6f3eff07dd0d39cb35f618b6835f8fb288a0f948 100644 (file)
--- a/src/os/bluestore/NVMEDevice.cc
+++ b/src/os/bluestore/NVMEDevice.cc
@@ -21,6 +21,12 @@
 #include <fcntl.h>
 #include <unistd.h>
 
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
+#include <rte_lcore.h>
+
 #include "include/types.h"
 #include "include/compat.h"
 #include "common/errno.h"
@@ -75,6 +81,12 @@ NVMEDevice::NVMEDevice(aio_callback_t cb, void *cbpriv)
   zeros.zero();
 }
 
+static char *ealargs[] = {
+    "perf",
+    "-c 0x1", /* This must be the second parameter. It is overwritten by index in main(). */
+    "-n 4",
+};
+
 int NVMEDevice::open(string p)
 {
   int r = 0;
@@ -82,6 +94,17 @@ int NVMEDevice::open(string p)
 
   pci_device *pci_dev;
 
+  r = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), (char **)(void *)(uintptr_t)ealargs);
+  if (r < 0) {
+    derr << __func__ << " init dpdk failed" << dendl;
+    return r;
+  }
+
+  if (request_mempool == NULL) {
+    derr << __func__ << " could not initialize request mempool" << dendl;
+    return -1;
+  }
+
   pci_system_init();
 
   // Search for matching devices
@@ -101,13 +124,11 @@ int NVMEDevice::open(string p)
 
   string sn_tag = g_conf->bdev_nvme_serial_number;
   if (sn_tag.empty()) {
-    int r = -ENOENT;
+    r = -ENOENT;
     derr << __func__ << " empty serial number: " << cpp_strerror(r) << dendl;
     return r;
   }
 
-  unbindfromkernel = g_conf->bdev_nvme_unbind_from_kernel;
-
   char serial_number[128];
   while ((pci_dev = pci_device_next(iter)) != NULL) {
     dout(10) << __func__ << " found device at "<< pci_dev->bus << ":" << pci_dev->dev << ":"
@@ -127,7 +148,7 @@ int NVMEDevice::open(string p)
     if (pci_device_has_kernel_driver(pci_dev)) {
       if (!pci_device_has_uio_driver(pci_dev)) {
         /*NVMe kernel driver case*/
-        if (unbindfromkernel) {
+        if (g_conf->bdev_nvme_unbind_from_kernel) {
           r =  pci_device_switch_to_uio_driver(pci_dev);
           if (r < 0) {
             derr << __func__ << " device " << pci_device_get_device_name(pci_dev) << " " << pci_dev->bus
@@ -174,13 +195,21 @@ int NVMEDevice::open(string p)
     if (num_ns > 1) {
       dout(0) << __func__ << " namespace count larger than 1, currently only use the first namespace" << dendl;
     }
-    nvme_namespace *ns = nvme_ctrlr_get_ns(ctrlr, 0);
+    ns = nvme_ctrlr_get_ns(ctrlr, 1);
+    if (!ns) {
+      derr << __func__ << " failed to get namespace at 1" << dendl;
+      return -1;
+    }
     block_size = nvme_ns_get_sector_size(ns);
     size = block_size * nvme_ns_get_num_sectors(ns);
     dout(1) << __func__ << " successfully attach nvme device at" << pci_device_get_device_name(pci_dev)
                         << " " << pci_dev->bus << ":" << pci_dev->dev << ":" << pci_dev->func << dendl;
     break;
   }
+  if (pci_dev == NULL) {
+    derr << __func__ << " failed to found nvme serial number " << sn_tag << dend;
+    return -ENOENT;
+  }
 
   pci_iterator_destroy(iter);
 

diff --git a/src/os/bluestore/NVMEDevice.h b/src/os/bluestore/NVMEDevice.h
index ace5d02cc7311d5c3cf217ab78f18f4361a52c4c..2d3871b4664f6f87fc90daaabca7e6850a7128bb 100644 (file)
--- a/src/os/bluestore/NVMEDevice.h
+++ b/src/os/bluestore/NVMEDevice.h
@@ -43,7 +43,6 @@ class NVMEDevice : public BlockDevice {
    */
   nvme_controller *ctrlr;
   nvme_namespace       *ns;
-  int unbindfromkernel = 0;
   string name;
 
   uint64_t size;