msg/async/rdma: add build ops

Signed-off-by: Haomai Wang <haomai@xsky.com>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 352fe463ad4e05df2a56f4dbe49b019e766878d9..98833fba25f21e4e05cb1e66ba3301d536dde935 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -180,6 +180,12 @@ else(ENABLE_SHARED)
 endif(ENABLE_SHARED)
 set(CMAKE_POSITION_INDEPENDENT_CODE ${ENABLE_SHARED})
 
+option(WITH_RDMA "Enable RDMA in async messenger" OFF)
+if(WITH_RDMA)
+  find_package(rdma REQUIRED)
+  set(HAVE_RDMA ${RDMA_FOUND})
+endif(WITH_RDMA)
+
 find_package(Backtrace)
 
 if(LINUX)

diff --git a/cmake/modules/Findrdma.cmake b/cmake/modules/Findrdma.cmake
new file mode 100644 (file)
index 0000000..2799247
--- /dev/null
+++ b/cmake/modules/Findrdma.cmake
@@ -0,0 +1,38 @@
+# - Find rdma
+# Find the rdma library and includes
+#
+# RDMA_INCLUDE_DIR - where to find ibverbs.h, etc.
+# RDMA_LIBRARIES - List of libraries when using ibverbs.
+# RDMA_FOUND - True if ibverbs found.
+
+find_path(RDMA_INCLUDE_DIR infiniband/verbs.h)
+
+set(RDMA_NAMES ${RDMA_NAMES} ibverbs)
+find_library(RDMA_LIBRARY NAMES ${RDMA_NAMES})
+
+if (RDMA_INCLUDE_DIR AND RDMA_LIBRARY)
+  set(RDMA_FOUND TRUE)
+  set(RDMA_LIBRARIES ${RDMA_LIBRARY})
+else ()
+  set(RDMA_FOUND FALSE)
+  set( RDMA_LIBRARIES )
+endif ()
+
+if (RDMA_FOUND)
+  message(STATUS "Found libibverbs: ${RDMA_LIBRARY}")
+else ()
+  message(STATUS "Not Found libibverbs: ${RDMA_LIBRARY}")
+  if (RDMA_FIND_REQUIRED)
+    message(STATUS "Looked for libibverbs named ${RDMA_NAMES}.")
+    message(FATAL_ERROR "Could NOT find libibverbs")
+  endif ()
+endif ()
+
+# handle the QUIETLY and REQUIRED arguments and set UUID_FOUND to TRUE if
+# all listed variables are TRUE
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ibverbs DEFAULT_MSG RDMA_LIBRARIES RDMA_INCLUDE_DIR)
+
+mark_as_advanced(
+  RDMA_LIBRARY
+)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c42af1d1208a4a437a6898e460f69e9ade442328..431a9945777db02ee2ebb7c5880dce210dd42e23 100644 (file)
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -226,6 +226,11 @@ if(HAVE_XIO)
   list(APPEND EXTRALIBS ${XIO_LIBRARY} pthread rt)
 endif(HAVE_XIO)
 
+if(HAVE_RDMA)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${RDMA_INCLUDE_DIR}")
+  list(APPEND EXTRALIBS ${RDMA_LIBRARIES} pthread rt)
+endif(HAVE_RDMA)
+
 # sort out which allocator to use
 if(ALLOCATOR STREQUAL "tcmalloc")
   set(ALLOC_LIBS ${GPERFTOOLS_TCMALLOC_LIBRARY})
@@ -309,6 +314,15 @@ if(HAVE_XIO)
     msg/xio/QueueStrategy.cc)
 endif(HAVE_XIO)
 
+set(async_rdma_common_srcs)
+if(HAVE_RDMA)
+  list(APPEND async_rdma_common_srcs
+    msg/async/rdma/Infiniband.cc
+    msg/async/rdma/RDMAConnectedSocketImpl.cc
+    msg/async/rdma/RDMAServerSocketImpl.cc
+    msg/async/rdma/RDMAStack.cc)
+endif(HAVE_RDMA)
+
 if(HAVE_GOOD_YASM_ELF64)
   set(yasm_srcs
     common/crc32c_intel_fast_asm.S
@@ -394,6 +408,7 @@ set(libcommon_files
   msg/async/PosixStack.cc
   msg/async/net_handler.cc
   ${xio_common_srcs}
+  ${async_rdma_common_srcs}
   msg/msg_types.cc
   common/hobject.cc
   osd/OSDMap.cc
@@ -543,7 +558,7 @@ endif (WITH_MGR)
 set(librados_config_srcs
   librados-config.cc)
 add_executable(librados-config ${librados_config_srcs})
-target_link_libraries(librados-config librados global ${BLKID_LIBRARIES}
+target_link_libraries(librados-config librados global ${BLKID_LIBRARIES} ${IBVERBS_LIBRARIES}
   ${CMAKE_DL_LIBS})
 
 install(TARGETS librados-config DESTINATION bin)
@@ -684,7 +699,7 @@ set(ceph_osd_srcs
 add_executable(ceph-osd ${ceph_osd_srcs}
   $<TARGET_OBJECTS:common_util_obj>)
 add_dependencies(ceph-osd erasure_code_plugins)
-target_link_libraries(ceph-osd osd os global ${BLKID_LIBRARIES})
+target_link_libraries(ceph-osd osd os global ${BLKID_LIBRARIES} ${IBVERBS_LIBRARIES})
 if(WITH_FUSE)
   target_link_libraries(ceph-osd ${FUSE_LIBRARIES})
 endif()

diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
index 1123f5812d6fbbfd825cc57eb0294cf909dbadae..80e9344fe5157a10b8d1783ec4afec422090e903 100644 (file)
--- a/src/include/config-h.in.cmake
+++ b/src/include/config-h.in.cmake
@@ -123,6 +123,9 @@
 /* Accelio conditional compilation */
 #cmakedefine HAVE_XIO
 
+/* AsyncMessenger RDMA conditional compilation */
+#cmakedefine HAVE_RDMA
+
 /* define if radosgw enabled */
 #cmakedefine WITH_RADOSGW