From b971cff51a9179c02f85a27cc191731a18e39876 Mon Sep 17 00:00:00 2001
From: Changcheng Liu <changcheng.liu@aliyun.com>
Date: Mon, 11 Nov 2019 16:11:38 +0800
Subject: [PATCH] msg/async/rdma: add configuration to select gid_idx

With Mellanox NIC, different gid_idx maps to different gid combined with
different RoCE protocol e.g. RoCEv1 or RoCEv2
1. Add configuration ms_async_rdma_gid_idx to set gid_idx.
1. Use "0" as default gid_idx to keep compatibility.
2. Initialize gid_idx in class member construction list.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>
---
 src/common/options.cc            | 4 ++++
 src/msg/async/rdma/Infiniband.cc | 5 +++--
 src/msg/async/rdma/Infiniband.h  | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index 1f76931f805..ee04e2ea8d8 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1148,6 +1148,10 @@ std::vector<Option> get_global_options() {
     .set_default(1000)
     .set_description(""),
 
+    Option("ms_async_rdma_gid_idx", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("use gid_idx to select GID for choosing RoCEv1 or RoCEv2"),
+
     Option("ms_async_rdma_local_gid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("")
     .set_description(""),
diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index fd229d43c62..b48597b1795 100644
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -30,7 +30,8 @@ static const uint32_t MAX_INLINE_DATA = 0;
 static const uint32_t TCP_MSG_LEN = sizeof("0000:00000000:00000000:00000000:00000000000000000000000000000000");
 static const uint32_t CQ_DEPTH = 30000;
 
-Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn)
+Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn),
+  gid_idx(cct->_conf.get_val<int64_t>("ms_async_rdma_gid_idx"))
 {
   int r = ibv_query_port(ctxt, port_num, &port_attr);
   if (r == -1) {
@@ -39,7 +40,7 @@ Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt
   }
 
   lid = port_attr.lid;
-
+  ceph_assert(gid_idx < port_attr.gid_tbl_len);
 #ifdef HAVE_IBV_EXP
   union ibv_gid cgid;
   struct ibv_exp_gid_attr gid_attr;
diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 4d5a14eb8d4..74dc51d97ee 100644
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -62,7 +62,7 @@ class Port {
   int port_num;
   struct ibv_port_attr port_attr;
   uint16_t lid;
-  int gid_idx = 0;
+  int gid_idx;
   union ibv_gid gid;
 
  public:
-- 
2.39.5