From b2ee4c277e45b5507cb763e9a701da3570b5012e Mon Sep 17 00:00:00 2001 From: Haomai Wang Date: Tue, 25 Apr 2017 23:58:00 +0800 Subject: [PATCH] msg/async/rdma: add inqueue rx chunks perf counter with rdma backend, we always spent a lot of time on why it stop receiving new message. most of probability is just lack of rx memory which may abused. Just make it clear how much rx memory inqueue! Signed-off-by: Haomai Wang --- src/msg/async/rdma/RDMAConnectedSocketImpl.cc | 5 +++++ src/msg/async/rdma/RDMAStack.cc | 5 ++++- src/msg/async/rdma/RDMAStack.h | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc index 34ea63279ee..3014bbbd4be 100644 --- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc +++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc @@ -111,10 +111,12 @@ RDMAConnectedSocketImpl::~RDMAConnectedSocketImpl() for (unsigned i=0; i < wc.size(); ++i) { ret = ibdev->post_chunk(reinterpret_cast(wc[i].wr_id)); assert(ret == 0); + dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks); } for (unsigned i=0; i < buffers.size(); ++i) { ret = ibdev->post_chunk(buffers[i]); assert(ret == 0); + dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks); } delete cmgr; @@ -355,6 +357,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len) ldout(cct, 20) << __func__ << " got remote close msg..." << dendl; } assert(ibdev->post_chunk(chunk) == 0); + dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks); } else { if (read == (ssize_t)len) { buffers.push_back(chunk); @@ -366,6 +369,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len) } else { read += chunk->read(buf+read, response->byte_len); assert(ibdev->post_chunk(chunk) == 0); + dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks); } } } @@ -388,6 +392,7 @@ ssize_t RDMAConnectedSocketImpl::read_buffers(char* buf, size_t len) ldout(cct, 25) << __func__ << " this iter read: " << tmp << " bytes." << " offset: " << (*c)->get_offset() << " ,bound: " << (*c)->get_bound() << ". Chunk:" << *c << dendl; if ((*c)->over()) { assert(ibdev->post_chunk(*c) == 0); + dispatcher->perf_logger->dec(l_msgr_rdma_inqueue_rx_chunks); ldout(cct, 25) << __func__ << " one chunk over." << dendl; } if (read == len) { diff --git a/src/msg/async/rdma/RDMAStack.cc b/src/msg/async/rdma/RDMAStack.cc index 88ce6bf168e..b1dae3bcf5c 100644 --- a/src/msg/async/rdma/RDMAStack.cc +++ b/src/msg/async/rdma/RDMAStack.cc @@ -52,6 +52,7 @@ RDMADispatcher::RDMADispatcher(CephContext* c, RDMAStack* s) plb.add_u64_counter(l_msgr_rdma_polling, "polling", "Whether dispatcher thread is polling"); plb.add_u64_counter(l_msgr_rdma_inflight_tx_chunks, "inflight_tx_chunks", "The number of inflight tx chunks"); + plb.add_u64_counter(l_msgr_rdma_inqueue_rx_chunks, "inqueue_rx_chunks", "The number of inqueue rx chunks"); plb.add_u64_counter(l_msgr_rdma_tx_total_wc, "tx_total_wc", "The number of tx work comletions"); plb.add_u64_counter(l_msgr_rdma_tx_total_wc_errors, "tx_total_wc_errors", "The number of tx errors"); @@ -179,8 +180,10 @@ void RDMADispatcher::polling() } } - for (auto &&i : polled) + for (auto &&i : polled) { + perf_logger->inc(l_msgr_rdma_inqueue_rx_chunks, i.second.size()); i.first->pass_wc(std::move(i.second)); + } polled.clear(); } diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h index 8e91d42f533..ef1188b8023 100644 --- a/src/msg/async/rdma/RDMAStack.h +++ b/src/msg/async/rdma/RDMAStack.h @@ -40,6 +40,7 @@ enum { l_msgr_rdma_polling, l_msgr_rdma_inflight_tx_chunks, + l_msgr_rdma_inqueue_rx_chunks, l_msgr_rdma_tx_total_wc, l_msgr_rdma_tx_total_wc_errors, -- 2.47.3