msg/async/rdma: use QueuePair as args to get more info

ibv_qp is member in class QueuePair. QueuePair has other fields
which is needed in post_chunks_to_rq to be further checked for
different hardware feature e.g. SRQ/iWARP/RoCE

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index b03d411a878c2342a8c8fb715348428e7da56edf..ca3b646cd1c2d19afd9f53631d393f3f05294b03 100644 (file)
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -1148,7 +1148,7 @@ Infiniband::QueuePair* Infiniband::create_queue_pair(CephContext *cct, Completio
   return qp;
 }
 
-int Infiniband::post_chunks_to_rq(int rq_wr_num, ibv_qp *qp)
+int Infiniband::post_chunks_to_rq(int rq_wr_num, QueuePair *qp)
 {
   int ret = 0;
   Chunk *chunk = nullptr;
@@ -1192,7 +1192,7 @@ int Infiniband::post_chunks_to_rq(int rq_wr_num, ibv_qp *qp)
     ret = ibv_post_srq_recv(srq, rx_work_request, &badworkrequest);
   } else {
     ceph_assert(qp);
-    ret = ibv_post_recv(qp, rx_work_request, &badworkrequest);
+    ret = ibv_post_recv(qp->get_qp(), rx_work_request, &badworkrequest);
   }
 
   ::free(rx_work_request);

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 2db1c0338ba35c14b35ad17ebeb85b0f16fe6803..57d5d98f781dcbc6ce28be0898b50be2cd09e37c 100644 (file)
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -523,7 +523,7 @@ class Infiniband {
       ibv_qp_type type, struct rdma_cm_id *cm_id);
   ibv_srq* create_shared_receive_queue(uint32_t max_wr, uint32_t max_sge);
   // post rx buffers to srq, return number of buffers actually posted
-  int post_chunks_to_rq(int num, ibv_qp *qp=NULL);
+  int post_chunks_to_rq(int num, QueuePair *qp = nullptr);
   void post_chunk_to_pool(Chunk* chunk) {
     get_memory_manager()->release_rx_buffer(chunk);
   }

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index 0b7b0bae15589225539083c5e0541d8a5367bf4f..c4fa58bcef9685c4775521c5e0daf38d4865d32c 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -576,11 +576,11 @@ void RDMAConnectedSocketImpl::set_accept_fd(int sd)
 
 void RDMAConnectedSocketImpl::post_chunks_to_rq(int num)
 {
-  post_backlog += num - ib->post_chunks_to_rq(num, qp->get_qp());
+  post_backlog += num - ib->post_chunks_to_rq(num, qp);
 }
 
 void RDMAConnectedSocketImpl::update_post_backlog()
 {
   if (post_backlog)
-    post_backlog -= post_backlog - dispatcher->post_chunks_to_rq(post_backlog, qp->get_qp());
+    post_backlog -= post_backlog - dispatcher->post_chunks_to_rq(post_backlog, qp);
 }

diff --git a/src/msg/async/rdma/RDMAIWARPConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAIWARPConnectedSocketImpl.cc
index a6ecbd5430c41e6221aef50c7a9da1e7ac549d1d..119a2131ed0a0db00fee70aabfe532dfb2362b6b 100644 (file)
--- a/src/msg/async/rdma/RDMAIWARPConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAIWARPConnectedSocketImpl.cc
@@ -163,7 +163,7 @@ int RDMAIWARPConnectedSocketImpl::alloc_resource() {
     return -1;
   }
   if (!cct->_conf->ms_async_rdma_support_srq)
-    dispatcher->post_chunks_to_rq(ib->get_rx_queue_len(), qp->get_qp());
+    dispatcher->post_chunks_to_rq(ib->get_rx_queue_len(), qp);
   dispatcher->register_qp(qp, this);
   dispatcher->perf_logger->inc(l_msgr_rdma_created_queue_pair);
   dispatcher->perf_logger->inc(l_msgr_rdma_active_queue_pair);

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index d04249fb4a8c034443d0b136f4710533d9ac5242..f761e1fbff9fed6426f5a698a3658ac6890570ec 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -239,7 +239,7 @@ void RDMADispatcher::post_chunk_to_pool(Chunk* chunk)
   perf_logger->dec(l_msgr_rdma_rx_bufs_in_use);
 }
 
-int RDMADispatcher::post_chunks_to_rq(int num, ibv_qp *qp)
+int RDMADispatcher::post_chunks_to_rq(int num, QueuePair *qp)
 {
   std::lock_guard l{lock};
   return ib->post_chunks_to_rq(num, qp);

diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index f553098a60f3e0d3d22b8eb3afbb4885f3ef9ee5..8cd8e194dd420b1f3a4e2ba7f7496fa4b78682ec 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -123,7 +123,7 @@ class RDMADispatcher {
   std::atomic<uint64_t> inflight = {0};
 
   void post_chunk_to_pool(Chunk* chunk);
-  int post_chunks_to_rq(int num, ibv_qp *qp=NULL);
+  int post_chunks_to_rq(int num, QueuePair *qp = nullptr);
 };
 
 class RDMAWorker : public Worker {