Addressing CR comments from tchaikov (Kefu Chai).

Signed-off-by: Yan Lei <yongyou.yl@alibaba-inc.com>

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index e774226b416d12f182666b636f881ac9634e9a28..db2245dd1e1dd37708fce86380926fc9980e6802 100644 (file)
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -166,9 +166,7 @@ Infiniband::QueuePair::QueuePair(
   max_send_wr(tx_queue_len),
   max_recv_wr(rx_queue_len),
   q_key(q_key),
-  dead(false),
-  tx_wr(0),
-  tx_wc(0)
+  dead(false)
 {
   initial_psn = lrand48() & 0xffffff;
   if (type != IBV_QPT_RC && type != IBV_QPT_UD && type != IBV_QPT_RAW_PACKET) {

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 7ebfbca73e281fe1c1de2ea21b930b86c94b8643..d46388659d3c82021dd446f9f2ad73ad0ce09c70 100644 (file)
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -465,14 +465,8 @@ class Infiniband {
      * Return true if the queue pair is in an error state, false otherwise.
      */
     bool is_error() const;
-    /**
-     * Add Tx work request and completion counters.
-     */
     void add_tx_wr(uint32_t amt) { tx_wr += amt; }
     void add_tx_wc(uint32_t amt) { tx_wc += amt; }
-    /**
-     * Get Tx work request and completion counter values.
-     */
     uint32_t get_tx_wr() const { return tx_wr; }
     uint32_t get_tx_wc() const { return tx_wc; }
     ibv_qp* get_qp() const { return qp; }
@@ -497,8 +491,8 @@ class Infiniband {
     uint32_t     max_recv_wr;
     uint32_t     q_key;
     bool dead;
-    std::atomic<uint32_t> tx_wr; // atomic counter for successful Tx WQEs
-    std::atomic<uint32_t> tx_wc; // atomic counter for successful Tx CQEs
+    std::atomic<uint32_t> tx_wr = {0}; // counter for successful Tx WQEs
+    std::atomic<uint32_t> tx_wc = {0}; // counter for successful Tx CQEs
   };
 
  public:

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index c9427f5bba6c6a6c2fe85630a8b18f4337a76200..ea489b8fea23277dc669826b6f0e69d992a1173f 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -577,7 +577,6 @@ int RDMAConnectedSocketImpl::post_work_request(std::vector<Chunk*> &tx_buffers)
     worker->perf_logger->inc(l_msgr_rdma_tx_failed);
     return -errno;
   }
-  // Update the Tx WQE counter
   qp->add_tx_wr(num);
   worker->perf_logger->inc(l_msgr_rdma_tx_chunks, tx_buffers.size());
   ldout(cct, 20) << __func__ << " qp state is " << Infiniband::qp_state_string(qp->get_state()) << dendl;
@@ -599,7 +598,6 @@ void RDMAConnectedSocketImpl::fin() {
     worker->perf_logger->inc(l_msgr_rdma_tx_failed);
     return ;
   }
-  // Update the Tx WQE counter
   qp->add_tx_wr(1);
 }
 

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index 031faf289a38e5e1c7bb58f2ea07f6d149ad761d..ce445add8f1eee60416a1de9084b122f7cc601ee 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -244,13 +244,15 @@ void RDMADispatcher::polling()
       perf_logger->set(l_msgr_rdma_inflight_tx_chunks, inflight);
       if (num_dead_queue_pair) {
         Mutex::Locker l(lock); // FIXME reuse dead qp because creating one qp costs 1 ms
-        for (auto idx = 0; idx < dead_queue_pairs.size(); idx++) {
+        for (auto &i : dead_queue_pairs) {
           // Bypass QPs that do not collect all Tx completions yet.
-          if (dead_queue_pairs.at(idx)->get_tx_wc() != dead_queue_pairs.at(idx)->get_tx_wr())
+          if (i->get_tx_wc() != i->get_tx_wr())
             continue;
-          ldout(cct, 10) << __func__ << " finally delete qp=" << dead_queue_pairs.at(idx) << dendl;
-          delete dead_queue_pairs.at(idx);
-          dead_queue_pairs.erase(dead_queue_pairs.begin() + idx);
+          ldout(cct, 10) << __func__ << " finally delete qp=" << i << dendl;
+          delete i;
+          auto it = std::find(dead_queue_pairs.begin(), dead_queue_pairs.end(), i);
+          if (it != dead_queue_pairs.end())
+            dead_queue_pairs.erase(it);
           perf_logger->dec(l_msgr_rdma_active_queue_pair);
           --num_dead_queue_pair;
         }
@@ -341,15 +343,15 @@ Infiniband::QueuePair* RDMADispatcher::get_qp(uint32_t qp)
   Mutex::Locker l(lock);
   // Try to find the QP in qp_conns firstly.
   auto it = qp_conns.find(qp);
-  if (it == qp_conns.end()) {
-    // Try again in dead_queue_pairs.
-    for(auto dead_qp = dead_queue_pairs.begin(); dead_qp != dead_queue_pairs.end(); dead_qp++) {
-      if ((*dead_qp)->get_local_qp_number() == qp)
-        return *dead_qp;
-    }
-    return nullptr;
-  }
-  return it->second.first;
+  if (it != qp_conns.end())
+    return it->second.first;
+
+  // Try again in dead_queue_pairs.
+  for (auto &i: dead_queue_pairs)
+    if (i->get_local_qp_number() == qp)
+      return i;
+
+  return nullptr;
 }
 
 void RDMADispatcher::erase_qpn_lockless(uint32_t qpn)