From 2d4890580f3acdd6387bcdde15f78eba35237589 Mon Sep 17 00:00:00 2001
From: Changcheng Liu <changcheng.liu@aliyun.com>
Date: Thu, 13 Jun 2019 18:34:44 +0800
Subject: [PATCH] msg/async/rdma: seperate Device construction if rdma_cm is
 used

If ms_async_rdma_cm is false, there's no need to call the api
rdma_get_device. If rdma_get_device is called, the devices remain
opened while librdmacm is loaded. This is not what we want when
ms_async_rdma_cm is false.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>
---
 src/msg/async/rdma/Infiniband.cc | 34 +++++++++++++++++++-------------
 src/msg/async/rdma/Infiniband.h  | 22 ++++++++++++++-------
 2 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index 0631d8c1490..ab38fcfb277 100644
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -103,24 +103,30 @@ Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt
 #endif
 }
 
-
-Device::Device(CephContext *cct, ibv_device* d, struct ibv_context *dc)
-  : device(d), active_port(nullptr)
+Device::Device(CephContext *cct, ibv_device* ib_dev): device(ib_dev), active_port(nullptr)
 {
-  if (device == NULL) {
-    lderr(cct) << __func__ << " device == NULL" << cpp_strerror(errno) << dendl;
-    ceph_abort();
-  }
+  ceph_assert(device);
+  ctxt = ibv_open_device(device);
+  ceph_assert(ctxt);
+
   name = ibv_get_device_name(device);
-  if (cct->_conf->ms_async_rdma_cm) {
-    ctxt = dc;
-  } else {
-    ctxt = ibv_open_device(device);
-  }
-  if (ctxt == NULL) {
-    lderr(cct) << __func__ << " open rdma device failed. " << cpp_strerror(errno) << dendl;
+
+  int r = ibv_query_device(ctxt, &device_attr);
+  if (r) {
+    lderr(cct) << __func__ << " failed to query rdma device. " << cpp_strerror(errno) << dendl;
     ceph_abort();
   }
+}
+
+Device::Device(CephContext *cct, struct ibv_context *ib_ctx): device(ib_ctx->device),
+                                                              active_port(nullptr)
+{
+  ceph_assert(device);
+  ctxt = ib_ctx;
+  ceph_assert(ctxt);
+
+  name = ibv_get_device_name(device);
+
   int r = ibv_query_device(ctxt, &device_attr);
   if (r) {
     lderr(cct) << __func__ << " failed to query rdma device. " << cpp_strerror(errno) << dendl;
diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index aaa972bd598..008ba187d2a 100644
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -78,7 +78,8 @@ class Device {
   const char* name;
   uint8_t  port_cnt = 0;
  public:
-  explicit Device(CephContext *c, ibv_device* d, struct ibv_context *dc);
+  explicit Device(CephContext *c, ibv_device* ib_dev);
+  explicit Device(CephContext *c, ibv_context *ib_ctx);
   ~Device() {
     if (active_port) {
       delete active_port;
@@ -102,16 +103,23 @@ class DeviceList {
   int num;
   Device** devices;
  public:
-  explicit DeviceList(CephContext *cct): device_list(ibv_get_device_list(&num)),
-                                device_context_list(rdma_get_devices(&num)) {
-    if (device_list == NULL || num == 0) {
-      lderr(cct) << __func__ << " failed to get rdma device list.  " << cpp_strerror(errno) << dendl;
-      ceph_abort();
+  explicit DeviceList(CephContext *cct): device_list(nullptr), device_context_list(nullptr),
+                                         num(0), devices(nullptr) {
+    device_list = ibv_get_device_list(&num);
+    ceph_assert(device_list);
+    ceph_assert(num);
+    if (cct->_conf->ms_async_rdma_cm) {
+        device_context_list = rdma_get_devices(NULL);
+        ceph_assert(device_context_list);
     }
     devices = new Device*[num];
 
     for (int i = 0;i < num; ++i) {
-      devices[i] = new Device(cct, device_list[i], device_context_list[i]);
+      if (cct->_conf->ms_async_rdma_cm) {
+         devices[i] = new Device(cct, device_context_list[i]);
+      } else {
+         devices[i] = new Device(cct, device_list[i]);
+      }
     }
   }
   ~DeviceList() {
-- 
2.39.5