mempool: pick shard on every allocation; move types to pool_t

If we have a static choice of shard for every *type*
(pool_allocator_base_t) then we will hammer that shard from all
threads and play cacheline ping-pong.  Instead, move the types
list to pool_t (there aren't that many anyway) and pick a shard
on every allocate/deallocate call.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/global/mempool.cc b/src/global/mempool.cc
index f5513d1fbab642b3c845177a0d25912484f232a8..614c8ce73275d1212b745ce64db02123d88f8224 100644 (file)
--- a/src/global/mempool.cc
+++ b/src/global/mempool.cc
@@ -91,18 +91,18 @@ void mempool::pool_t::get_stats(
   for (size_t i = 0; i < num_shards; ++i) {
     total->items += shard[i].items;
     total->bytes += shard[i].bytes;
-    if (debug) {
-      std::unique_lock<std::mutex> shard_lock(shard[i].lock);
-      for (const list_member_t *p = shard[i].types.next;
-          p != &shard[i].types;
-          p = p->next) {
-       const pool_allocator_base_t *c =
-         reinterpret_cast<const pool_allocator_base_t *>(p);
-       std::string n = ceph_demangle(c->type_id);
-       stats_t &s = (*by_type)[n];
-       s.bytes = c->items * c->item_size;
-       s.items = c->items;
-      }
+  }
+  if (debug) {
+    std::unique_lock<std::mutex> shard_lock(lock);
+    for (const list_member_t *p = types.next;
+        p != &types;
+        p = p->next) {
+      const pool_allocator_base_t *c =
+       reinterpret_cast<const pool_allocator_base_t *>(p);
+      std::string n = ceph_demangle(c->type_id);
+      stats_t &s = (*by_type)[n];
+      s.bytes = c->items * c->item_size;
+      s.items = c->items;
     }
   }
 }

diff --git a/src/include/mempool.h b/src/include/mempool.h
index 522753a933e9ef5c5a569e14ff37245c73ede31e..8612db0fb8c2364c4fc83021fa9043551b43f232 100644 (file)
--- a/src/include/mempool.h
+++ b/src/include/mempool.h
@@ -223,8 +223,6 @@ enum {
 struct shard_t {
   std::atomic<size_t> bytes = {0};
   std::atomic<size_t> items = {0};
-  mutable std::mutex lock;  // only used for types list
-  list_member_t types;      // protected by lock
 };
 
 struct stats_t {
@@ -242,7 +240,6 @@ struct pool_allocator_base_t {
   list_member_t list_member;   // this must come first; see get_stats() hackery
 
   pool_t *pool = nullptr;
-  shard_t *shard = nullptr;
   const char *type_id = nullptr;
   size_t item_size = 0;
 
@@ -260,6 +257,10 @@ pool_t& get_pool(pool_index_t ix);
 class pool_t {
   std::string name;
   shard_t shard[num_shards];
+
+  mutable std::mutex lock;  // only used for types list
+  list_member_t types;      // protected by lock
+
   friend class pool_allocator_base_t;
 public:
   bool debug;
@@ -302,18 +303,17 @@ inline void pool_allocator_base_t::attach_pool(
 {
   assert(pool == nullptr);
   pool = &get_pool(index);
-  shard = pool->pick_a_shard();
   type_id = _type_id;
 
   // unconditionally register type, even if debug is currently off
-  std::unique_lock<std::mutex> lock(shard->lock);
-  shard->types.insert(&list_member);
+  std::unique_lock<std::mutex> lock(pool->lock);
+  pool->types.insert(&list_member);
 }
 
 inline pool_allocator_base_t::~pool_allocator_base_t()
 {
   if (pool) {
-    std::unique_lock<std::mutex> lock(shard->lock);
+    std::unique_lock<std::mutex> lock(pool->lock);
     list_member.remove();
   }
 }
@@ -357,8 +357,9 @@ public:
 
   pointer allocate(size_t n, void *p = nullptr) {
     size_t total = sizeof(T) * n;
-    base.shard->bytes += total;
-    base.shard->items += n;
+    shard_t *shard = base.pool->pick_a_shard();
+    shard->bytes += total;
+    shard->items += n;
     if (base.pool->debug) {
       base.items += n;
     }
@@ -368,8 +369,9 @@ public:
 
   void deallocate(pointer p, size_type n) {
     size_t total = sizeof(T) * n;
-    base.shard->bytes -= total;
-    base.shard->items -= n;
+    shard_t *shard = base.pool->pick_a_shard();
+    shard->bytes -= total;
+    shard->items -= n;
     if (base.pool->debug) {
       base.items -= n;
     }