msg/async/rdma: implement modify_qp_to_init to init qp

For IB/RoCE, the QP need go through "reset->init->ready to receive
->ready to send" for normal operation under most cases.
This patch implement seperate function to initialize qp.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index 844861678b00b8fcc907ebbd0a857f0bdfb20e30..23fc5ff31d2e88a128f0a884b0a8776114df089c 100644 (file)
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -182,6 +182,42 @@ Infiniband::QueuePair::QueuePair(
   }
 }
 
+int Infiniband::QueuePair::modify_qp_to_init(void)
+{
+  // move from RESET to INIT state
+  ibv_qp_attr qpa;
+  memset(&qpa, 0, sizeof(qpa));
+  qpa.qp_state   = IBV_QPS_INIT;
+  qpa.pkey_index = 0;
+  qpa.port_num   = (uint8_t)(ib_physical_port);
+  qpa.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
+  qpa.qkey       = q_key;
+
+  int mask = IBV_QP_STATE | IBV_QP_PORT;
+  switch (type) {
+    case IBV_QPT_RC:
+      mask |= IBV_QP_ACCESS_FLAGS;
+      mask |= IBV_QP_PKEY_INDEX;
+      break;
+    case IBV_QPT_UD:
+      mask |= IBV_QP_QKEY;
+      mask |= IBV_QP_PKEY_INDEX;
+      break;
+    case IBV_QPT_RAW_PACKET:
+      break;
+    default:
+      ceph_abort();
+  }
+
+  if (ibv_modify_qp(qp, &qpa, mask)) {
+    lderr(cct) << __func__ << " failed to switch to INIT state Queue Pair, qp number: " << qp->qp_num
+               << " Error: " << cpp_strerror(errno) << dendl;
+    return -1;
+  }
+  ldout(cct, 20) << __func__ << " successfully switch to INIT state Queue Pair, qp number: " << qp->qp_num << dendl;
+  return 0;
+}
+
 int Infiniband::QueuePair::init()
 {
   ldout(cct, 20) << __func__ << " started." << dendl;
@@ -212,6 +248,10 @@ int Infiniband::QueuePair::init()
       }
       return -1;
     }
+    if (modify_qp_to_init() != 0) {
+      ibv_destroy_qp(qp);
+      return -1;
+    }
   } else {
     ceph_assert(cm_id->verbs == pd->context);
     if (rdma_create_qp(cm_id, pd, &qpia)) {
@@ -223,44 +263,6 @@ int Infiniband::QueuePair::init()
   }
   ldout(cct, 20) << __func__ << " successfully create queue pair: "
                  << "qp=" << qp << dendl;
-
-  if (cct->_conf->ms_async_rdma_cm)
-    return 0;
-
-  // move from RESET to INIT state
-  ibv_qp_attr qpa;
-  memset(&qpa, 0, sizeof(qpa));
-  qpa.qp_state   = IBV_QPS_INIT;
-  qpa.pkey_index = 0;
-  qpa.port_num   = (uint8_t)(ib_physical_port);
-  qpa.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
-  qpa.qkey       = q_key;
-
-  int mask = IBV_QP_STATE | IBV_QP_PORT;
-  switch (type) {
-    case IBV_QPT_RC:
-      mask |= IBV_QP_ACCESS_FLAGS;
-      mask |= IBV_QP_PKEY_INDEX;
-      break;
-    case IBV_QPT_UD:
-      mask |= IBV_QP_QKEY;
-      mask |= IBV_QP_PKEY_INDEX;
-      break;
-    case IBV_QPT_RAW_PACKET:
-      break;
-    default:
-      ceph_abort();
-  }
-
-  int ret = ibv_modify_qp(qp, &qpa, mask);
-  if (ret) {
-    ibv_destroy_qp(qp);
-    lderr(cct) << __func__ << " failed to transition to INIT state: "
-               << cpp_strerror(errno) << dendl;
-    return -1;
-  }
-  ldout(cct, 20) << __func__ << " successfully change queue pair to INIT:"
-                 << " qp=" << qp << dendl;
   return 0;
 }
 

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 7b1da1f2ce95e85b58ff0162b22c279256fb49e7..63ac44c2c3e6469d4cec5b4b93372af119d49024 100644 (file)
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -446,6 +446,7 @@ class Infiniband {
               uint32_t tx_queue_len, uint32_t max_recv_wr, struct rdma_cm_id *cid, uint32_t q_key = 0);
     ~QueuePair();
 
+    int modify_qp_to_init();
     int init();
 
     /**