buffer: allow buffers to map into arbitrary mempools

- default is buffer_anon
- buffer_data includes *all* buffers across all mempools, and is the
*actual* allocation (includes raw_combined).
- buffer_meta is other buffer-related overhead.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/buffer.cc b/src/common/buffer.cc
index 6910a441bef7586e28e4a7ae734a6418726e3144..42c150f45fade48eb2e8419b1f438741613ce628 100644 (file)
--- a/src/common/buffer.cc
+++ b/src/common/buffer.cc
@@ -168,17 +168,46 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT;
     char *data;
     unsigned len;
     atomic_t nref;
+    int mempool = mempool::mempool_buffer_anon;
 
     mutable std::atomic_flag crc_spinlock = ATOMIC_FLAG_INIT;
     map<pair<size_t, size_t>, pair<uint32_t, uint32_t> > crc_map;
 
     explicit raw(unsigned l)
-      : data(NULL), len(l), nref(0)
-    { }
+      : data(NULL), len(l), nref(0) {
+      mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len);
+    }
     raw(char *c, unsigned l)
-      : data(c), len(l), nref(0)
-    { }
-    virtual ~raw() {}
+      : data(c), len(l), nref(0) {
+      mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len);
+    }
+    virtual ~raw() {
+      mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(
+       -1, -(int)len);
+    }
+
+    void _set_len(unsigned l) {
+      mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(
+       -1, -(int)len);
+      len = l;
+      mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len);
+    }
+
+    void reassign_to_mempool(int pool) {
+      if (pool == mempool) {
+       return;
+      }
+      mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(
+       -1, -(int)len);
+      mempool = pool;
+      mempool::get_pool(mempool::pool_index_t(pool)).adjust_count(1, len);
+    }
+
+    void try_assign_to_mempool(int pool) {
+      if (mempool == mempool::mempool_buffer_anon) {
+       reassign_to_mempool(pool);
+      }
+    }
 
     // no copying.
     // cppcheck-suppress noExplicitConstructor
@@ -462,7 +491,7 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT;
        return r;
       }
       // update length with actual amount read
-      len = r;
+      _set_len(r);
       return 0;
     }
 
@@ -1455,6 +1484,7 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT;
   {
     std::swap(_len, other._len);
     std::swap(_memcopy_count, other._memcopy_count);
+    std::swap(_mempool, other._mempool);
     _buffers.swap(other._buffers);
     append_buffer.swap(other.append_buffer);
     //last_p.swap(other.last_p);
@@ -1627,6 +1657,28 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT;
     return is_aligned(CEPH_PAGE_SIZE);
   }
 
+  void buffer::list::reassign_to_mempool(int pool)
+  {
+    _mempool = pool;
+    if (append_buffer.get_raw()) {
+      append_buffer.get_raw()->reassign_to_mempool(pool);
+    }
+    for (auto& p : _buffers) {
+      p.get_raw()->reassign_to_mempool(pool);
+    }
+  }
+
+  void buffer::list::try_assign_to_mempool(int pool)
+  {
+    _mempool = pool;
+    if (append_buffer.get_raw()) {
+      append_buffer.get_raw()->try_assign_to_mempool(pool);
+    }
+    for (auto& p : _buffers) {
+      p.get_raw()->try_assign_to_mempool(pool);
+    }
+  }
+
   void buffer::list::rebuild()
   {
     if (_len == 0) {
@@ -1714,6 +1766,17 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT;
    return  rebuild_aligned(CEPH_PAGE_SIZE);
   }
 
+  void buffer::list::reserve(size_t prealloc)
+  {
+    if (append_buffer.unused_tail_length() < prealloc) {
+      append_buffer = buffer::create(prealloc);
+      if (_mempool) {
+       append_buffer.get_raw()->reassign_to_mempool(_mempool);
+      }
+      append_buffer.set_length(0);   // unused, so far.
+    }
+  }
+
   // sort-of-like-assignment-op
   void buffer::list::claim(list& bl, unsigned int flags)
   {

diff --git a/src/include/buffer.h b/src/include/buffer.h
index 9a79f1fe6a2296a3b2a344520f8786d4f1b9eb72..c998fe324ebbc13c7e918303dccabbfcac1fe760 100644 (file)
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -348,6 +348,7 @@ namespace buffer CEPH_BUFFER_API {
     unsigned _len;
     unsigned _memcopy_count; //the total of memcopy using rebuild().
     ptr append_buffer;  // where i put small appends.
+    int _mempool = -1;
 
   public:
     class iterator;
@@ -681,6 +682,7 @@ namespace buffer CEPH_BUFFER_API {
       _memcopy_count = other._memcopy_count;
       last_p = begin();
       append_buffer.swap(other.append_buffer);
+      _mempool = other._mempool;
       other.clear();
       return *this;
     }
@@ -689,6 +691,9 @@ namespace buffer CEPH_BUFFER_API {
     const ptr& front() const { return _buffers.front(); }
     const ptr& back() const { return _buffers.back(); }
 
+    void reassign_to_mempool(int pool);
+    void try_assign_to_mempool(int pool);
+
     size_t get_append_buffer_unused_tail_length() const {
       return append_buffer.unused_tail_length();
     }
@@ -774,12 +779,7 @@ namespace buffer CEPH_BUFFER_API {
                                         unsigned align_memory);
     bool rebuild_page_aligned();
 
-    void reserve(size_t prealloc) {
-      if (append_buffer.unused_tail_length() < prealloc) {
-       append_buffer = buffer::create(prealloc);
-       append_buffer.set_length(0);   // unused, so far.
-      }
-    }
+    void reserve(size_t prealloc);
 
     // assignment-op with move semantics
     const static unsigned int CLAIM_DEFAULT = 0;

diff --git a/src/include/mempool.h b/src/include/mempool.h
index 4914313a3009bfbbebbfac7f80c48d1b18ee5879..5eaf6b3d46a696db1012daeea58bd4b41d1c180c 100644 (file)
--- a/src/include/mempool.h
+++ b/src/include/mempool.h
@@ -142,8 +142,9 @@ namespace mempool {
   f(bluestore_alloc)                 \
   f(bluestore_fsck)                  \
   f(bluefs)                          \
-  f(buffer_meta)                     \
+  f(buffer_anon)                     \
   f(buffer_data)                     \
+  f(buffer_meta)                     \
   f(osd)                             \
   f(osdmap)                          \
   f(osdmap_mapping)