From 32da5f1d038bb9b8abb8cc7fccd7df3f3e4ceef4 Mon Sep 17 00:00:00 2001
From: Changcheng Liu <changcheng.liu@aliyun.com>
Date: Mon, 1 Jul 2019 10:27:45 +0800
Subject: [PATCH] msg/async/rdma: use different strategy to reset read/write
 chunk

When releasing read chunk to pool, the chunk::offset & chunk::bound
should be reset to zero. For write chunk, it's better to reset
chunk::offset to zero and chunk::bound to chunk length which means that
[offset, bound) is writable.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>
---
 src/msg/async/rdma/Infiniband.cc              | 10 ++++++++--
 src/msg/async/rdma/Infiniband.h               |  3 ++-
 src/msg/async/rdma/RDMAConnectedSocketImpl.cc |  6 +++---
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index 4a87657853d0b..f86f3b5cdfed1 100644
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -561,12 +561,18 @@ bool Infiniband::MemoryManager::Chunk::full()
   return offset == bytes;
 }
 
-void Infiniband::MemoryManager::Chunk::clear()
+void Infiniband::MemoryManager::Chunk::reset_read_chunk()
 {
   offset = 0;
   bound = 0;
 }
 
+void Infiniband::MemoryManager::Chunk::reset_write_chunk()
+{
+  offset = 0;
+  bound = bytes;
+}
+
 Infiniband::MemoryManager::Cluster::Cluster(MemoryManager& m, uint32_t s)
   : manager(m), buffer_size(s)
 {
@@ -612,7 +618,7 @@ void Infiniband::MemoryManager::Cluster::take_back(std::vector<Chunk*> &ck)
 {
   std::lock_guard l{lock};
   for (auto c : ck) {
-    c->clear();
+    c->reset_write_chunk();
     free_chunks.push_back(c);
   }
 }
diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 2ddaecf8b3fb6..5292a1dbbe3c7 100644
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -215,7 +215,8 @@ class Infiniband {
       uint32_t read(char* buf, uint32_t len);
       uint32_t write(char* buf, uint32_t len);
       bool full();
-      void clear();
+      void reset_read_chunk();
+      void reset_write_chunk();
 
      public:
       ibv_mr* mr;
diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index e60cd9b363158..9764257bf362a 100644
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -297,7 +297,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
     Chunk* chunk = reinterpret_cast<Chunk *>(response->wr_id);
     chunk->prepare_read(response->byte_len);
     if (chunk->get_size() == 0) {
-      chunk->clear();
+      chunk->reset_read_chunk();
       dispatcher->perf_logger->inc(l_msgr_rdma_rx_fin);
       if (connected) {
         error = ECONNRESET;
@@ -315,7 +315,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
       buffers.push_back(chunk);
       ldout(cct, 25) << __func__ << " buffers add a chunk: " << chunk->get_offset() << ":" << chunk->get_bound() << dendl;
     } else {
-      chunk->clear();
+      chunk->reset_read_chunk();
       dispatcher->post_chunk_to_pool(chunk);
       update_post_backlog();
     }
@@ -349,7 +349,7 @@ ssize_t RDMAConnectedSocketImpl::read_buffers(char* buf, size_t len)
                    << (*pchunk)->get_offset() << " ,bound: " << (*pchunk)->get_bound() << dendl;
 
     if ((*pchunk)->get_size() == 0) {
-      (*pchunk)->clear();
+      (*pchunk)->reset_read_chunk();
       dispatcher->post_chunk_to_pool(*pchunk);
       update_post_backlog();
       ldout(cct, 25) << __func__ << " read over one chunk " << dendl;
-- 
2.39.5