msg/async/rdma: add configuration to select gid_idx

With Mellanox NIC, different gid_idx maps to different gid combined with
different RoCE protocol e.g. RoCEv1 or RoCEv2
1. Add configuration ms_async_rdma_gid_idx to set gid_idx.
1. Use "0" as default gid_idx to keep compatibility.
2. Initialize gid_idx in class member construction list.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index 1f76931f8054c25c68765cba07468f1811bb56ff..ee04e2ea8d84252979d51c68f56575c5a589094f 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1148,6 +1148,10 @@ std::vector<Option> get_global_options() {
     .set_default(1000)
     .set_description(""),
 
+    Option("ms_async_rdma_gid_idx", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("use gid_idx to select GID for choosing RoCEv1 or RoCEv2"),
+
     Option("ms_async_rdma_local_gid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("")
     .set_description(""),

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index fd229d43c62b18bb61646bef9bd3de6e8c2978cf..b48597b17951c4919ce69e4e247a38babadefe3b 100644 (file)
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -30,7 +30,8 @@ static const uint32_t MAX_INLINE_DATA = 0;
 static const uint32_t TCP_MSG_LEN = sizeof("0000:00000000:00000000:00000000:00000000000000000000000000000000");
 static const uint32_t CQ_DEPTH = 30000;
 
-Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn)
+Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn),
+  gid_idx(cct->_conf.get_val<int64_t>("ms_async_rdma_gid_idx"))
 {
   int r = ibv_query_port(ctxt, port_num, &port_attr);
   if (r == -1) {
@@ -39,7 +40,7 @@ Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt
   }
 
   lid = port_attr.lid;
-
+  ceph_assert(gid_idx < port_attr.gid_tbl_len);
 #ifdef HAVE_IBV_EXP
   union ibv_gid cgid;
   struct ibv_exp_gid_attr gid_attr;

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 4d5a14eb8d4cac24ec7da87dd1a125369317ff96..74dc51d97eea14ba4f58e14286a10b2d0c421c42 100644 (file)
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -62,7 +62,7 @@ class Port {
   int port_num;
   struct ibv_port_attr port_attr;
   uint16_t lid;
-  int gid_idx = 0;
+  int gid_idx;
   union ibv_gid gid;
 
  public: