From: ownedu <yanlei_cv@aliyun.com>
Date: Sun, 1 Oct 2017 09:07:53 +0000 (+0800)
Subject: Addressing CR comments from tchaikov (Kefu Chai).
X-Git-Tag: v13.0.1~696^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=303e640c74c2a6f1907664b25d8d9d577ec210bf;p=ceph.git

Addressing CR comments from tchaikov (Kefu Chai).

Signed-off-by: Yan Lei <yongyou.yl@alibaba-inc.com>
---

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index e774226b416d..db2245dd1e1d 100644
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -166,9 +166,7 @@ Infiniband::QueuePair::QueuePair(
   max_send_wr(tx_queue_len),
   max_recv_wr(rx_queue_len),
   q_key(q_key),
-  dead(false),
-  tx_wr(0),
-  tx_wc(0)
+  dead(false)
 {
   initial_psn = lrand48() & 0xffffff;
   if (type != IBV_QPT_RC && type != IBV_QPT_UD && type != IBV_QPT_RAW_PACKET) {
diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 7ebfbca73e28..d46388659d3c 100644
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -465,14 +465,8 @@ class Infiniband {
      * Return true if the queue pair is in an error state, false otherwise.
      */
     bool is_error() const;
-    /**
-     * Add Tx work request and completion counters.
-     */
     void add_tx_wr(uint32_t amt) { tx_wr += amt; }
     void add_tx_wc(uint32_t amt) { tx_wc += amt; }
-    /**
-     * Get Tx work request and completion counter values.
-     */
     uint32_t get_tx_wr() const { return tx_wr; }
     uint32_t get_tx_wc() const { return tx_wc; }
     ibv_qp* get_qp() const { return qp; }
@@ -497,8 +491,8 @@ class Infiniband {
     uint32_t     max_recv_wr;
     uint32_t     q_key;
     bool dead;
-    std::atomic<uint32_t> tx_wr; // atomic counter for successful Tx WQEs
-    std::atomic<uint32_t> tx_wc; // atomic counter for successful Tx CQEs
+    std::atomic<uint32_t> tx_wr = {0}; // counter for successful Tx WQEs
+    std::atomic<uint32_t> tx_wc = {0}; // counter for successful Tx CQEs
   };
 
  public:
diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index c9427f5bba6c..ea489b8fea23 100644
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -577,7 +577,6 @@ int RDMAConnectedSocketImpl::post_work_request(std::vector<Chunk*> &tx_buffers)
     worker->perf_logger->inc(l_msgr_rdma_tx_failed);
     return -errno;
   }
-  // Update the Tx WQE counter
   qp->add_tx_wr(num);
   worker->perf_logger->inc(l_msgr_rdma_tx_chunks, tx_buffers.size());
   ldout(cct, 20) << __func__ << " qp state is " << Infiniband::qp_state_string(qp->get_state()) << dendl;
@@ -599,7 +598,6 @@ void RDMAConnectedSocketImpl::fin() {
     worker->perf_logger->inc(l_msgr_rdma_tx_failed);
     return ;
   }
-  // Update the Tx WQE counter
   qp->add_tx_wr(1);
 }
 
diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index 031faf289a38..ce445add8f1e 100644
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -244,13 +244,15 @@ void RDMADispatcher::polling()
       perf_logger->set(l_msgr_rdma_inflight_tx_chunks, inflight);
       if (num_dead_queue_pair) {
         Mutex::Locker l(lock); // FIXME reuse dead qp because creating one qp costs 1 ms
-        for (auto idx = 0; idx < dead_queue_pairs.size(); idx++) {
+        for (auto &i : dead_queue_pairs) {
           // Bypass QPs that do not collect all Tx completions yet.
-          if (dead_queue_pairs.at(idx)->get_tx_wc() != dead_queue_pairs.at(idx)->get_tx_wr())
+          if (i->get_tx_wc() != i->get_tx_wr())
             continue;
-          ldout(cct, 10) << __func__ << " finally delete qp=" << dead_queue_pairs.at(idx) << dendl;
-          delete dead_queue_pairs.at(idx);
-          dead_queue_pairs.erase(dead_queue_pairs.begin() + idx);
+          ldout(cct, 10) << __func__ << " finally delete qp=" << i << dendl;
+          delete i;
+          auto it = std::find(dead_queue_pairs.begin(), dead_queue_pairs.end(), i);
+          if (it != dead_queue_pairs.end())
+            dead_queue_pairs.erase(it);
           perf_logger->dec(l_msgr_rdma_active_queue_pair);
           --num_dead_queue_pair;
         }
@@ -341,15 +343,15 @@ Infiniband::QueuePair* RDMADispatcher::get_qp(uint32_t qp)
   Mutex::Locker l(lock);
   // Try to find the QP in qp_conns firstly.
   auto it = qp_conns.find(qp);
-  if (it == qp_conns.end()) {
-    // Try again in dead_queue_pairs.
-    for(auto dead_qp = dead_queue_pairs.begin(); dead_qp != dead_queue_pairs.end(); dead_qp++) {
-      if ((*dead_qp)->get_local_qp_number() == qp)
-        return *dead_qp;
-    }
-    return nullptr;
-  }
-  return it->second.first;
+  if (it != qp_conns.end())
+    return it->second.first;
+
+  // Try again in dead_queue_pairs.
+  for (auto &i: dead_queue_pairs)
+    if (i->get_local_qp_number() == qp)
+      return i;
+
+  return nullptr;
 }
 
 void RDMADispatcher::erase_qpn_lockless(uint32_t qpn)