From: Haomai Wang <haomai@xsky.com>
Date: Tue, 25 Apr 2017 15:58:00 +0000 (+0800)
Subject: msg/async/rdma: add inqueue rx chunks perf counter
X-Git-Tag: v12.0.3~210^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b2ee4c277e45b5507cb763e9a701da3570b5012e;p=ceph.git

msg/async/rdma: add inqueue rx chunks perf counter

with rdma backend, we always spent a lot of time on why it stop receiving
new message. most of probability is just lack of rx memory which may abused.

Just make it clear how much rx memory inqueue!

Signed-off-by: Haomai Wang <haomai@xsky.com>
---

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index 34ea63279ee0..3014bbbd4be6 100644
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -111,10 +111,12 @@ RDMAConnectedSocketImpl::~RDMAConnectedSocketImpl()
   for (unsigned i=0; i < wc.size(); ++i) {
     ret = ibdev->post_chunk(reinterpret_cast<Chunk*>(wc[i].wr_id));
     assert(ret == 0);
+    dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
   }
   for (unsigned i=0; i < buffers.size(); ++i) {
     ret = ibdev->post_chunk(buffers[i]);
     assert(ret == 0);
+    dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
   }
 
   delete cmgr;
@@ -355,6 +357,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
         ldout(cct, 20) << __func__ << " got remote close msg..." << dendl;
       }
       assert(ibdev->post_chunk(chunk) == 0);
+      dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
     } else {
       if (read == (ssize_t)len) {
         buffers.push_back(chunk);
@@ -366,6 +369,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
       } else {
         read += chunk->read(buf+read, response->byte_len);
         assert(ibdev->post_chunk(chunk) == 0);
+        dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
       }
     }
   }
@@ -388,6 +392,7 @@ ssize_t RDMAConnectedSocketImpl::read_buffers(char* buf, size_t len)
     ldout(cct, 25) << __func__ << " this iter read: " << tmp << " bytes." << " offset: " << (*c)->get_offset() << " ,bound: " << (*c)->get_bound()  << ". Chunk:" << *c  << dendl;
     if ((*c)->over()) {
       assert(ibdev->post_chunk(*c) == 0);
+      dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks);
       ldout(cct, 25) << __func__ << " one chunk over." << dendl;
     }
     if (read == len) {
diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc
index 88ce6bf168eb..b1dae3bcf5c2 100644
--- a/src/msg/async/rdma/RDMAStack.cc
+++ b/src/msg/async/rdma/RDMAStack.cc
@@ -52,6 +52,7 @@ RDMADispatcher::RDMADispatcher(CephContext* c, RDMAStack* s)
 
   plb.add_u64_counter(l_msgr_rdma_polling, "polling", "Whether dispatcher thread is polling");
   plb.add_u64_counter(l_msgr_rdma_inflight_tx_chunks, "inflight_tx_chunks", "The number of inflight tx chunks");
+  plb.add_u64_counter(l_msgr_rdma_inqueue_rx_chunks, "inqueue_rx_chunks", "The number of inqueue rx chunks");
 
   plb.add_u64_counter(l_msgr_rdma_tx_total_wc, "tx_total_wc", "The number of tx work comletions");
   plb.add_u64_counter(l_msgr_rdma_tx_total_wc_errors, "tx_total_wc_errors", "The number of tx errors");
@@ -179,8 +180,10 @@ void RDMADispatcher::polling()
         }
       }
 
-      for (auto &&i : polled)
+      for (auto &&i : polled) {
+        perf_logger->inc(l_msgr_rdma_inqueue_rx_chunks, i.second.size());
         i.first->pass_wc(std::move(i.second));
+      }
       polled.clear();
     }
 
diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index 8e91d42f533b..ef1188b80237 100644
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -40,6 +40,7 @@ enum {
 
   l_msgr_rdma_polling,
   l_msgr_rdma_inflight_tx_chunks,
+  l_msgr_rdma_inqueue_rx_chunks,
 
   l_msgr_rdma_tx_total_wc,
   l_msgr_rdma_tx_total_wc_errors,