msg/async/rdma: add inqueue rx chunks perf counter

with rdma backend, we always spent a lot of time on why it stop receiving
new message. most of probability is just lack of rx memory which may abused.

Just make it clear how much rx memory inqueue!

Signed-off-by: Haomai Wang <haomai@xsky.com>

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index 34ea63279ee0eda26700fc9c33591d8674e3b87a..3014bbbd4be60e2a909c9b676cdd0f272c1d46b8 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -111,10 +111,12 @@ RDMAConnectedSocketImpl::~RDMAConnectedSocketImpl()
   for (unsigned i=0; i < wc.size(); ++i) {
     ret = ibdev->post_chunk(reinterpret_cast<Chunk*>(wc[i].wr_id));
     assert(ret == 0);
+    dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
   }
   for (unsigned i=0; i < buffers.size(); ++i) {
     ret = ibdev->post_chunk(buffers[i]);
     assert(ret == 0);
+    dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
   }
 
   delete cmgr;
@@ -355,6 +357,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
         ldout(cct, 20) << __func__ << " got remote close msg..." << dendl;
       }
       assert(ibdev->post_chunk(chunk) == 0);
+      dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
     } else {
       if (read == (ssize_t)len) {
         buffers.push_back(chunk);
@@ -366,6 +369,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
       } else {
         read += chunk->read(buf+read, response->byte_len);
         assert(ibdev->post_chunk(chunk) == 0);
+        dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
       }
     }
   }
@@ -388,6 +392,7 @@ ssize_t RDMAConnectedSocketImpl::read_buffers(char* buf, size_t len)
     ldout(cct, 25) << __func__ << " this iter read: " << tmp << " bytes." << " offset: " << (*c)->get_offset() << " ,bound: " << (*c)->get_bound()  << ". Chunk:" << *c  << dendl;
     if ((*c)->over()) {
       assert(ibdev->post_chunk(*c) == 0);
+      dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
       ldout(cct, 25) << __func__ << " one chunk over." << dendl;
     }
     if (read == len) {

diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index 88ce6bf168ebf0094bee83c3d3884c99774f5d9c..b1dae3bcf5c2f02fa46d8ba9e4efc974c77e3391 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -52,6 +52,7 @@ RDMADispatcher::RDMADispatcher(CephContext* c, RDMAStack* s)
 
   plb.add_u64_counter(l_msgr_rdma_polling, "polling", "Whether dispatcher thread is polling");
   plb.add_u64_counter(l_msgr_rdma_inflight_tx_chunks, "inflight_tx_chunks", "The number of inflight tx chunks");
+  plb.add_u64_counter(l_msgr_rdma_inqueue_rx_chunks, "inqueue_rx_chunks", "The number of inqueue rx chunks");
 
   plb.add_u64_counter(l_msgr_rdma_tx_total_wc, "tx_total_wc", "The number of tx work comletions");
   plb.add_u64_counter(l_msgr_rdma_tx_total_wc_errors, "tx_total_wc_errors", "The number of tx errors");
@@ -179,8 +180,10 @@ void RDMADispatcher::polling()
         }
       }
 
-      for (auto &&i : polled)
+      for (auto &&i : polled) {
+        perf_logger->inc(l_msgr_rdma_inqueue_rx_chunks, i.second.size());
         i.first->pass_wc(std::move(i.second));
+      }
       polled.clear();
     }
 

diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index 8e91d42f533b0ad52e576db0026a652a1512ce65..ef1188b802375e3538ff7b7b0e3fdfdaaa26ee7d 100644 (file)
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -40,6 +40,7 @@ enum {
 
   l_msgr_rdma_polling,
   l_msgr_rdma_inflight_tx_chunks,
+  l_msgr_rdma_inqueue_rx_chunks,
 
   l_msgr_rdma_tx_total_wc,
   l_msgr_rdma_tx_total_wc_errors,