From: Changcheng Liu <changcheng.liu@aliyun.com>
Date: Thu, 27 Jun 2019 03:03:44 +0000 (+0800)
Subject: msg/async/rdma: refine Chunk construction function
X-Git-Tag: v15.1.0~1781^2~16
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6a0d3df90ac20076534e9849ecef0b90dbf64dff;p=ceph.git

msg/async/rdma: refine Chunk construction function

1. all values are initialized in construction function
   In this way, it's easy to construct Chunk object in
   PoolAllocator::malloc function.
2. For read chunk, member bound is initialized to be 0.
3. For send chunk, member bound is initialzied to be full space size.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>
---

diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index 3bb207d971ac..3702e7b576d1 100644
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -498,8 +498,9 @@ Infiniband::ProtectionDomain::~ProtectionDomain()
 }
 
 
-Infiniband::MemoryManager::Chunk::Chunk(ibv_mr* m, uint32_t len, char* b)
-  : mr(m), bytes(len), offset(0), buffer(b)
+Infiniband::MemoryManager::Chunk::Chunk(ibv_mr* m, uint32_t bytes, char* buffer,
+    uint32_t offset, uint32_t bound, uint32_t lkey)
+  : mr(m), lkey(lkey), bytes(bytes), offset(offset), bound(bound), buffer(buffer)
 {
 }
 
@@ -600,7 +601,7 @@ int Infiniband::MemoryManager::Cluster::fill(uint32_t num)
   ceph_assert(m);
   Chunk* chunk = chunk_base;
   for (uint32_t offset = 0; offset < bytes; offset += buffer_size){
-    new(chunk) Chunk(m, buffer_size, base+offset);
+    new(chunk) Chunk(m, buffer_size, base + offset, 0, buffer_size, m->lkey);
     free_chunks.push_back(chunk);
     chunk++;
   }
@@ -726,10 +727,7 @@ char *Infiniband::MemoryManager::PoolAllocator::malloc(const size_type bytes)
   /* initialize chunks */
   ch = m->chunks;
   for (unsigned i = 0; i < nbufs; i++) {
-    ch->lkey   = m->mr->lkey;
-    ch->bytes  = cct->_conf->ms_async_rdma_buffer_size;
-    ch->offset = 0;
-    ch->buffer = ch->data; // TODO: refactor tx and remove buffer
+    new(ch) Chunk(m->mr, cct->_conf->ms_async_rdma_buffer_size, ch->data, 0, 0, m->mr->lkey);
     ch = reinterpret_cast<Chunk *>(reinterpret_cast<char *>(ch) + rx_buf_size);
   }
 
diff --git a/src/msg/async/rdma/Infiniband.h b/src/msg/async/rdma/Infiniband.h
index 81967a4b7bc9..2ddaecf8b3fb 100644
--- a/src/msg/async/rdma/Infiniband.h
+++ b/src/msg/async/rdma/Infiniband.h
@@ -203,7 +203,7 @@ class Infiniband {
    public:
     class Chunk {
      public:
-      Chunk(ibv_mr* m, uint32_t len, char* b);
+      Chunk(ibv_mr* m, uint32_t bytes, char* buffer, uint32_t offset = 0, uint32_t bound = 0, uint32_t lkey = 0);
       ~Chunk();
 
       void set_offset(uint32_t o);
@@ -219,10 +219,10 @@ class Infiniband {
 
      public:
       ibv_mr* mr;
-      uint32_t lkey = 0;
+      uint32_t lkey;
       uint32_t bytes;
-      uint32_t bound = 0;
       uint32_t offset;
+      uint32_t bound;
       char* buffer; // TODO: remove buffer/refactor TX
       char  data[0];
     };