msg/async/rdma: check if exp verbs avail

issue: 975125

Change-Id: I9daa168ca0299887a7238a688508c773b98abde9
Signed-off-by: Adir Lev <adirl@mellanox.com>
Signed-off-by: Oren Duer <oren@mellanox.com>

diff --git a/cmake/modules/Findrdma.cmake b/cmake/modules/Findrdma.cmake
index 2799247ba1f0acde0959b0c9e3640594b55a6660..eb31f7922c91c2124d02c03f843da87261e7e4cb 100644 (file)
--- a/cmake/modules/Findrdma.cmake
+++ b/cmake/modules/Findrdma.cmake
@@ -20,6 +20,17 @@ endif ()
 
 if (RDMA_FOUND)
   message(STATUS "Found libibverbs: ${RDMA_LIBRARY}")
+
+  include(CheckCXXSourceCompiles)
+  CHECK_CXX_SOURCE_COMPILES("
+    #include <infiniband/verbs.h>
+    int main() {
+      struct ibv_context* ctxt;
+      struct ibv_exp_gid_attr gid_attr;
+      ibv_exp_query_gid_attr(ctxt, 1, 0, &gid_attr);
+      return 0;
+    } " HAVE_IBV_EXP)
+
 else ()
   message(STATUS "Not Found libibverbs: ${RDMA_LIBRARY}")
   if (RDMA_FIND_REQUIRED)

diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
index d731a9b6c1f0e4c9caf42db216b7ff97572d8cdc..575daed610bac14e432a0875c713690e3f887d22 100644 (file)
--- a/src/include/config-h.in.cmake
+++ b/src/include/config-h.in.cmake
@@ -130,6 +130,9 @@
 /* AsyncMessenger RDMA conditional compilation */
 #cmakedefine HAVE_RDMA
 
+/* ibverbs experimental conditional compilation */
+#cmakedefine HAVE_IBV_EXP
+
 /* define if embedded enabled */
 #cmakedefine WITH_EMBEDDED
 

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index e6d9cd297da5d0bb08077507aefbcefb1364b8b4..40a69bcacfa4789b781213e5de5f997204528d9a 100644 (file)
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -29,10 +29,12 @@ static const uint32_t CQ_DEPTH = 30000;
 
 Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn), port_attr(new ibv_port_attr)
 {
+#ifdef HAVE_IBV_EXP
   union ibv_gid cgid;
   struct ibv_exp_gid_attr gid_attr;
   bool malformed = false;
 
+  ldout(cct,1) << __func__ << " using experimental verbs for gid" << dendl;
   int r = ibv_query_port(ctxt, port_num, port_attr);
   if (r == -1) {
     lderr(cct) << __func__  << " query port failed  " << cpp_strerror(errno) << dendl;
@@ -87,6 +89,20 @@ Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt
     lderr(cct) << __func__ << " Requested local GID was not found in GID table" << dendl;
     ceph_abort();
   }
+#else
+  int r = ibv_query_port(ctxt, port_num, port_attr);
+  if (r == -1) {
+    lderr(cct) << __func__  << " query port failed  " << cpp_strerror(errno) << dendl;
+    ceph_abort();
+  }
+
+  lid = port_attr->lid;
+  r = ibv_query_gid(ctxt, port_num, 0, &gid);
+  if (r) {
+    lderr(cct) << __func__  << " query gid failed  " << cpp_strerror(errno) << dendl;
+    ceph_abort();
+  }
+#endif
 }