msg/async/rdma: use different strategy to reset read/write chunk

When releasing read chunk to pool, the chunk::offset & chunk::bound
should be reset to zero. For write chunk, it's better to reset
chunk::offset to zero and chunk::bound to chunk length which means that
[offset, bound) is writable.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index 4a87657853d0bf5107119b0a5992fc473e686d79..f86f3b5cdfed11fb3e99ffba5beed5def506c7cf 100644 (file)
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -561,12 +561,18 @@ bool Infiniband::MemoryManager::Chunk::full()
   return offset == bytes;
 }
 
-void Infiniband::MemoryManager::Chunk::clear()
+void Infiniband::MemoryManager::Chunk::reset_read_chunk()
 {
   offset = 0;
   bound = 0;
 }
 
+void Infiniband::MemoryManager::Chunk::reset_write_chunk()
+{
+  offset = 0;
+  bound = bytes;
+}
+
 Infiniband::MemoryManager::Cluster::Cluster(MemoryManager& m, uint32_t s)
   : manager(m), buffer_size(s)
 {
@@ -612,7 +618,7 @@ void Infiniband::MemoryManager::Cluster::take_back(std::vector<Chunk*> &ck)
 {
   std::lock_guard l{lock};
   for (auto c : ck) {
-    c->clear();
+    c->reset_write_chunk();
     free_chunks.push_back(c);
   }
 }

diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 2ddaecf8b3fb60fa170ac829e91c1cc58544cd56..5292a1dbbe3c740002f7e716324ddec4e586e59a 100644 (file)
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -215,7 +215,8 @@ class Infiniband {
       uint32_t read(char* buf, uint32_t len);
       uint32_t write(char* buf, uint32_t len);
       bool full();
-      void clear();
+      void reset_read_chunk();
+      void reset_write_chunk();
 
      public:
       ibv_mr* mr;

diff --git a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
index e60cd9b3631584484deacb1980901e22863c9b27..9764257bf362a685920034bf93521dc3454c0aea 100644 (file)
--- a/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
+++ b/src/msg/async/rdma/RDMAConnectedSocketImpl.cc
@@ -297,7 +297,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
     Chunk* chunk = reinterpret_cast<Chunk *>(response->wr_id);
     chunk->prepare_read(response->byte_len);
     if (chunk->get_size() == 0) {
-      chunk->clear();
+      chunk->reset_read_chunk();
       dispatcher->perf_logger->inc(l_msgr_rdma_rx_fin);
       if (connected) {
         error = ECONNRESET;
@@ -315,7 +315,7 @@ ssize_t RDMAConnectedSocketImpl::read(char* buf, size_t len)
       buffers.push_back(chunk);
       ldout(cct, 25) << __func__ << " buffers add a chunk: " << chunk->get_offset() << ":" << chunk->get_bound() << dendl;
     } else {
-      chunk->clear();
+      chunk->reset_read_chunk();
       dispatcher->post_chunk_to_pool(chunk);
       update_post_backlog();
     }
@@ -349,7 +349,7 @@ ssize_t RDMAConnectedSocketImpl::read_buffers(char* buf, size_t len)
                    << (*pchunk)->get_offset() << " ,bound: " << (*pchunk)->get_bound() << dendl;
 
     if ((*pchunk)->get_size() == 0) {
-      (*pchunk)->clear();
+      (*pchunk)->reset_read_chunk();
       dispatcher->post_chunk_to_pool(*pchunk);
       update_post_backlog();
       ldout(cct, 25) << __func__ << " read over one chunk " << dendl;