Addressing CR comments from alex-mikheev (Alex Mikheev), to use a single

atomic counter for inflight Tx CQEs.

Signed-off-by: Yan Lei <yongyou.yl@alibaba-inc.com>

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index d46388659d3c82021dd446f9f2ad73ad0ce09c70..b8d3789c99964817d46fd2bb9e5ee158202a3069 100644 (file)
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -465,10 +465,9 @@ class Infiniband {
      * Return true if the queue pair is in an error state, false otherwise.
      */
     bool is_error() const;
-    void add_tx_wr(uint32_t amt) { tx_wr += amt; }
-    void add_tx_wc(uint32_t amt) { tx_wc += amt; }
-    uint32_t get_tx_wr() const { return tx_wr; }
-    uint32_t get_tx_wc() const { return tx_wc; }
+    void add_tx_wr(uint32_t amt) { tx_wr_inflight += amt; }
+    void dec_tx_wr(uint32_t amt) { tx_wr_inflight -= amt; }
+    uint32_t get_tx_wr() const { return tx_wr_inflight; }
     ibv_qp* get_qp() const { return qp; }
     Infiniband::CompletionQueue* get_tx_cq() const { return txcq; }
     Infiniband::CompletionQueue* get_rx_cq() const { return rxcq; }
@@ -491,8 +490,7 @@ class Infiniband {
     uint32_t     max_recv_wr;
     uint32_t     q_key;
     bool dead;
-    std::atomic<uint32_t> tx_wr = {0}; // counter for successful Tx WQEs
-    std::atomic<uint32_t> tx_wc = {0}; // counter for successful Tx CQEs
+    std::atomic<uint32_t> tx_wr_inflight = {0}; // counter for inflight Tx WQEs
   };
 
  public:

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index ce445add8f1eee60416a1de9084b122f7cc601ee..080abb6d07efad47c6363ec5b5352e1b88970f56 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -246,7 +246,7 @@ void RDMADispatcher::polling()
         Mutex::Locker l(lock); // FIXME reuse dead qp because creating one qp costs 1 ms
         for (auto &i : dead_queue_pairs) {
           // Bypass QPs that do not collect all Tx completions yet.
-          if (i->get_tx_wc() != i->get_tx_wr())
+          if (i->get_tx_wr())
             continue;
           ldout(cct, 10) << __func__ << " finally delete qp=" << i << dendl;
           delete i;
@@ -382,10 +382,9 @@ void RDMADispatcher::handle_tx_event(ibv_wc *cqe, int n)
                    << " len: " << response->byte_len << " , addr:" << chunk
                    << " " << get_stack()->get_infiniband().wc_status_to_string(response->status) << dendl;
 
-    // Update the Tx CQE counter.
     QueuePair *qp = get_qp(response->qp_num);
     if (qp)
-      qp->add_tx_wc(1);
+      qp->dec_tx_wr(1);
 
     if (response->status != IBV_WC_SUCCESS) {
       perf_logger->inc(l_msgr_rdma_tx_total_wc_errors);