From e3237714e4af1844596d97dcc0a75ffc1e942673 Mon Sep 17 00:00:00 2001 From: ownedu Date: Sun, 1 Oct 2017 18:27:52 +0800 Subject: [PATCH] Addressing CR comments from alex-mikheev (Alex Mikheev), to use a single atomic counter for inflight Tx CQEs. Signed-off-by: Yan Lei --- src/msg/async/rdma/Infiniband.h | 10 ++++------ src/msg/async/rdma/RDMAStack.cc | 5 ++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h index d46388659d3c8..b8d3789c99964 100644 --- a/src/msg/async/rdma/Infiniband.h +++ b/src/msg/async/rdma/Infiniband.h @@ -465,10 +465,9 @@ class Infiniband { * Return true if the queue pair is in an error state, false otherwise. */ bool is_error() const; - void add_tx_wr(uint32_t amt) { tx_wr += amt; } - void add_tx_wc(uint32_t amt) { tx_wc += amt; } - uint32_t get_tx_wr() const { return tx_wr; } - uint32_t get_tx_wc() const { return tx_wc; } + void add_tx_wr(uint32_t amt) { tx_wr_inflight += amt; } + void dec_tx_wr(uint32_t amt) { tx_wr_inflight -= amt; } + uint32_t get_tx_wr() const { return tx_wr_inflight; } ibv_qp* get_qp() const { return qp; } Infiniband::CompletionQueue* get_tx_cq() const { return txcq; } Infiniband::CompletionQueue* get_rx_cq() const { return rxcq; } @@ -491,8 +490,7 @@ class Infiniband { uint32_t max_recv_wr; uint32_t q_key; bool dead; - std::atomic tx_wr = {0}; // counter for successful Tx WQEs - std::atomic tx_wc = {0}; // counter for successful Tx CQEs + std::atomic tx_wr_inflight = {0}; // counter for inflight Tx WQEs }; public: diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc index ce445add8f1ee..080abb6d07efa 100644 --- a/src/msg/async/rdma/RDMAStack.cc +++ b/src/msg/async/rdma/RDMAStack.cc @@ -246,7 +246,7 @@ void RDMADispatcher::polling() Mutex::Locker l(lock); // FIXME reuse dead qp because creating one qp costs 1 ms for (auto &i : dead_queue_pairs) { // Bypass QPs that do not collect all Tx completions yet. - if (i->get_tx_wc() != i->get_tx_wr()) + if (i->get_tx_wr()) continue; ldout(cct, 10) << __func__ << " finally delete qp=" << i << dendl; delete i; @@ -382,10 +382,9 @@ void RDMADispatcher::handle_tx_event(ibv_wc *cqe, int n) << " len: " << response->byte_len << " , addr:" << chunk << " " << get_stack()->get_infiniband().wc_status_to_string(response->status) << dendl; - // Update the Tx CQE counter. QueuePair *qp = get_qp(response->qp_num); if (qp) - qp->add_tx_wc(1); + qp->dec_tx_wr(1); if (response->status != IBV_WC_SUCCESS) { perf_logger->inc(l_msgr_rdma_tx_total_wc_errors); -- 2.39.5