os/bluestore: measure allocator lock wait and processing latency

Fixes: https://tracker.ceph.com/issues/76936
Signed-off-by: Jaya Prakash <jayaprakash@ibm.com>

diff --git a/src/os/bluestore/AvlAllocator.cc b/src/os/bluestore/AvlAllocator.cc
index b6a0325b5ebe10f03fb7b73f92d3fd832825e817..13d08a08821230d18d5b09956bb919d043af783e 100644 (file)
--- a/src/os/bluestore/AvlAllocator.cc
+++ b/src/os/bluestore/AvlAllocator.cc
@@ -368,6 +368,7 @@ AvlAllocator::AvlAllocator(CephContext* cct,
                            uint64_t max_mem,
                            std::string_view name) :
   AllocatorBase(name, device_size, block_size),
+  AllocatorPerf(cct, name),
   cct(cct),
   range_size_alloc_threshold(
     cct->_conf.get_val<uint64_t>("bluestore_avl_alloc_bf_threshold")),
@@ -421,8 +422,22 @@ int64_t AvlAllocator::allocate(
       max_alloc_size >= cap) {
     max_alloc_size = p2align(uint64_t(cap), (uint64_t)block_size);
   }
+  auto lock_wait_start = mono_clock::now();
+
   std::lock_guard l(lock);
-  return _allocate(want, unit, max_alloc_size, hint, extents);
+
+  auto lock_acquired = mono_clock::now();
+
+  auto ret = _allocate(want, unit, max_alloc_size, hint, extents);
+
+  logger->tinc_with_max(
+      l_bluestore_allocator_alloc_process_lat,
+      mono_clock::now() - lock_acquired);
+  logger->tinc_with_max(
+      l_bluestore_allocator_lock_wait_lat,
+      lock_acquired - lock_wait_start);
+
+  return ret;
 }
 
 void AvlAllocator::release(const release_set_t& release_set) {

diff --git a/src/os/bluestore/AvlAllocator.h b/src/os/bluestore/AvlAllocator.h
index b0a9fdd0cfbeccda83eb74bb37e095db5127caed..c2c86acb20400eb683cac7bb4fdfaf2cc81a6115 100644 (file)
--- a/src/os/bluestore/AvlAllocator.h
+++ b/src/os/bluestore/AvlAllocator.h
@@ -50,7 +50,7 @@ struct range_seg_t {
   boost::intrusive::avl_set_member_hook<> size_hook;
 };
 
-class AvlAllocator : public AllocatorBase {
+class AvlAllocator : public AllocatorBase, public AllocatorPerf {
   struct dispose_rs {
     void operator()(range_seg_t* p)
     {

diff --git a/src/os/bluestore/Btree2Allocator.cc b/src/os/bluestore/Btree2Allocator.cc
index 2775c7347f6292711b0264d4b16026dee971bcce..7531f059852d4ede065078f3a70e4206f1ea0af0 100644 (file)
--- a/src/os/bluestore/Btree2Allocator.cc
+++ b/src/os/bluestore/Btree2Allocator.cc
@@ -24,6 +24,7 @@ Btree2Allocator::Btree2Allocator(CephContext* _cct,
   bool with_cache,
   std::string_view name) :
     AllocatorBase(name, device_size, block_size),
+    AllocatorPerf(_cct, name),
     myTraits(RANGE_SIZE_BUCKET_COUNT),
     cct(_cct),
     range_count_cap(max_mem / sizeof(range_seg_t))
@@ -90,8 +91,22 @@ int64_t Btree2Allocator::allocate(
     extents->emplace_back(cached_chunk_offs, want);
     return want;
   }
+  auto lock_wait_start = mono_clock::now();
+
   std::lock_guard l(lock);
-  return _allocate(want, unit, max_alloc_size, hint, extents);
+
+  auto lock_acquired = mono_clock::now();
+
+  auto ret = _allocate(want, unit, max_alloc_size, hint, extents);
+
+  logger->tinc_with_max(
+      l_bluestore_allocator_alloc_process_lat,
+      mono_clock::now() - lock_acquired);
+  logger->tinc_with_max(
+      l_bluestore_allocator_lock_wait_lat,
+      lock_acquired - lock_wait_start);
+
+  return ret;
 }
 
 void Btree2Allocator::release(const release_set_t& release_set)

diff --git a/src/os/bluestore/Btree2Allocator.h b/src/os/bluestore/Btree2Allocator.h
index 5aa5b5bee5c981be59016b85acf0517588666291..6e6a54405b5e99f834432c86eacf5693bf89bec3 100644 (file)
--- a/src/os/bluestore/Btree2Allocator.h
+++ b/src/os/bluestore/Btree2Allocator.h
@@ -20,7 +20,7 @@
  *
  *
  */
-class Btree2Allocator : public AllocatorBase {
+class Btree2Allocator : public AllocatorBase, public AllocatorPerf {
   enum {
     RANGE_SIZE_BUCKET_COUNT = 14,
   };

diff --git a/src/os/bluestore/BtreeAllocator.cc b/src/os/bluestore/BtreeAllocator.cc
index 5a7ecc4ad104f50f09905ec265fab49a73ef335a..629c7a07349666af3fc4842dab2b99bde7479fe0 100644 (file)
--- a/src/os/bluestore/BtreeAllocator.cc
+++ b/src/os/bluestore/BtreeAllocator.cc
@@ -354,6 +354,7 @@ BtreeAllocator::BtreeAllocator(CephContext* cct,
                               uint64_t max_mem,
                               std::string_view name) :
   AllocatorBase(name, device_size, block_size),
+  AllocatorPerf(cct, name),
   range_size_alloc_threshold(
     cct->_conf.get_val<uint64_t>("bluestore_avl_alloc_bf_threshold")),
   range_size_alloc_free_pct(
@@ -397,8 +398,22 @@ int64_t BtreeAllocator::allocate(
       max_alloc_size >= cap) {
     max_alloc_size = p2align(uint64_t(cap), (uint64_t)block_size);
   }
+  auto lock_wait_start = mono_clock::now();
+
   std::lock_guard l(lock);
-  return _allocate(want, unit, max_alloc_size, hint, extents);
+
+  auto lock_acquired = mono_clock::now();
+
+  auto ret = _allocate(want, unit, max_alloc_size, hint, extents);
+
+  logger->tinc_with_max(
+      l_bluestore_allocator_alloc_process_lat,
+      mono_clock::now() - lock_acquired);
+  logger->tinc_with_max(
+      l_bluestore_allocator_lock_wait_lat,
+      lock_acquired - lock_wait_start);
+
+  return ret;
 }
 
 void BtreeAllocator::release(const interval_set<uint64_t>& release_set) {

diff --git a/src/os/bluestore/BtreeAllocator.h b/src/os/bluestore/BtreeAllocator.h
index d4328169bdd2064c153b531ad78da05f4a9a5c90..897c06ec3726ff0d794acb1635b8af623ff95c9f 100644 (file)
--- a/src/os/bluestore/BtreeAllocator.h
+++ b/src/os/bluestore/BtreeAllocator.h
@@ -11,7 +11,7 @@
 #include "os/bluestore/bluestore_types.h"
 #include "include/mempool.h"
 
-class BtreeAllocator : public AllocatorBase {
+class BtreeAllocator : public AllocatorBase, public AllocatorPerf {
   struct range_seg_t {
     uint64_t start;   ///< starting offset of this segment
     uint64_t end;     ///< ending offset (non-inclusive)

diff --git a/src/os/bluestore/HybridAllocator_impl.h b/src/os/bluestore/HybridAllocator_impl.h
index 590ee49bb3ec502d6c7a470eacc3aba2e9b6a6c3..6d1053abae43d908fd426caf56ee666108dcf54b 100644 (file)
--- a/src/os/bluestore/HybridAllocator_impl.h
+++ b/src/os/bluestore/HybridAllocator_impl.h
@@ -35,8 +35,12 @@ int64_t HybridAllocatorBase<T>::allocate(
     max_alloc_size = p2align(uint64_t(cap), (uint64_t)T::get_block_size());
   }
 
+  auto lock_wait_start = mono_clock::now();
+
   std::lock_guard l(T::get_lock());
 
+  auto lock_acquired = mono_clock::now();
+
   // try bitmap first to avoid unneeded contiguous extents split if
   // desired amount is less than shortes range in AVL or Btree2
   bool primary_first = !(bmap_alloc &&
@@ -63,6 +67,12 @@ int64_t HybridAllocatorBase<T>::allocate(
       ceph_assert(orig_size == extents->size());
     }
   }
+  this->logger->tinc_with_max(
+      l_bluestore_allocator_alloc_process_lat,
+      mono_clock::now() - lock_acquired);
+  this->logger->tinc_with_max(
+      l_bluestore_allocator_lock_wait_lat,
+      lock_acquired - lock_wait_start);
   return res ? res : -ENOSPC;
 }
 

diff --git a/src/os/bluestore/StupidAllocator.cc b/src/os/bluestore/StupidAllocator.cc
index 1533cf2987035201c7299ddf5b4780fd61491190..4b908cb7f5e14f8d687574a0215196a9bae85d82 100644 (file)
--- a/src/os/bluestore/StupidAllocator.cc
+++ b/src/os/bluestore/StupidAllocator.cc
@@ -15,6 +15,7 @@ StupidAllocator::StupidAllocator(CephContext* cct,
                                  int64_t _block_size,
                                  std::string_view name)
   : AllocatorBase(name, capacity, _block_size),
+    AllocatorPerf(cct, name),
     cct(cct), num_free(0),
     free(10)
 {
@@ -56,7 +57,12 @@ int64_t StupidAllocator::allocate_int(
   uint64_t want_size, uint64_t alloc_unit, int64_t hint,
   uint64_t *offset, uint32_t *length)
 {
+  auto lock_wait_start = mono_clock::now();
+
   std::lock_guard l(lock);
+
+  auto lock_acquired = mono_clock::now();
+
   ldout(cct, 10) << __func__ << " want_size 0x" << std::hex << want_size
                 << " alloc_unit 0x" << alloc_unit
                 << " hint 0x" << hint << std::dec
@@ -164,6 +170,13 @@ int64_t StupidAllocator::allocate_int(
   num_free -= *length;
   ceph_assert(num_free >= 0);
   last_alloc = *offset + *length;
+
+  logger->tinc_with_max(
+      l_bluestore_allocator_alloc_process_lat,
+      mono_clock::now() - lock_acquired);
+  logger->tinc_with_max(
+      l_bluestore_allocator_lock_wait_lat,
+      lock_acquired - lock_wait_start);
   return 0;
 }
 

diff --git a/src/os/bluestore/StupidAllocator.h b/src/os/bluestore/StupidAllocator.h
index fe75e90bd9d34f17fe6d0bfdf9089dc877f1a9c4..3f0af4efdb5ec9e4be533460cddf2875f648af62 100644 (file)
--- a/src/os/bluestore/StupidAllocator.h
+++ b/src/os/bluestore/StupidAllocator.h
@@ -14,7 +14,7 @@
 #include "include/mempool.h"
 #include "common/ceph_mutex.h"
 
-class StupidAllocator : public AllocatorBase {
+class StupidAllocator : public AllocatorBase, public AllocatorPerf {
   CephContext* cct;
   ceph::mutex lock = ceph::make_mutex("StupidAllocator::lock");