os/bluestore: add perf counter for allocator fragmentation.

Signed-off-by: Igor Fedotov <ifedotov@suse.com>
(cherry picked from commit 9a2cbaed8a70936d2c2eae69083c39b78358dc90)

diff --git a/src/os/bluestore/Allocator.h b/src/os/bluestore/Allocator.h
index 370e7e132c09dff09756b969ecff476540e0de42..7e4e8648ef3d68b269b84561a1bdbf42f8037b64 100644 (file)
--- a/src/os/bluestore/Allocator.h
+++ b/src/os/bluestore/Allocator.h
@@ -53,6 +53,10 @@ public:
   virtual void init_rm_free(uint64_t offset, uint64_t length) = 0;
 
   virtual uint64_t get_free() = 0;
+  virtual double get_fragmentation(uint64_t alloc_unit)
+  {
+    return 0.0;
+  }
 
   virtual void shutdown() = 0;
   static Allocator *create(CephContext* cct, string type, int64_t size,

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 5bd4e870faac08f2e4e292b00852a83e16b5737d..6a32c53b0f2edc74a15d42458631cddbc3814b14 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -4192,6 +4192,8 @@ void BlueStore::_init_logger()
                     "Read EIO errors propagated to high level callers");
   b.add_u64_counter(l_bluestore_reads_with_retries, "bluestore_reads_with_retries",
                     "Read operations that required at least one retry due to failed checksum validation");
+  b.add_u64(l_bluestore_fragmentation, "bluestore_fragmentation_micros",
+            "How fragmented bluestore free space is (free extents / max possible number of free extents) * 1000");
   logger = b.create_perf_counters();
   cct->get_perfcounters_collection()->add(logger);
 }
@@ -8522,6 +8524,8 @@ void BlueStore::_txc_finish(TransContext *txc)
     dout(10) << __func__ << " reaping empty zombie osr " << osr << dendl;
     osr->_unregister();
   }
+  logger->set(l_bluestore_fragmentation,
+    (uint64_t)(alloc->get_fragmentation(min_alloc_size) * 1000));
 }
 
 void BlueStore::_txc_release_alloc(TransContext *txc)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 66d9d4bd15b746449e14f174fde077d26c2a5164..916054010d9b236667e71549d121bab61d7df42d 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -119,6 +119,7 @@ enum {
   l_bluestore_gc_merged,
   l_bluestore_read_eio,
   l_bluestore_reads_with_retries,
+  l_bluestore_fragmentation,
   l_bluestore_last
 };
 

diff --git a/src/os/bluestore/StupidAllocator.cc b/src/os/bluestore/StupidAllocator.cc
index ead4641baf2d06965890c6fe65d1c2baa00eab1f..13a7daa2274bbb9f439bd615c1cf98f2dbdb57f2 100644 (file)
--- a/src/os/bluestore/StupidAllocator.cc
+++ b/src/os/bluestore/StupidAllocator.cc
@@ -272,6 +272,31 @@ uint64_t StupidAllocator::get_free()
   return num_free;
 }
 
+double StupidAllocator::get_fragmentation(uint64_t alloc_unit)
+{
+  assert(alloc_unit);
+  double res;
+  uint64_t max_intervals = 0;
+  uint64_t intervals = 0;
+  {
+    std::lock_guard<std::mutex> l(lock);
+    max_intervals = num_free / alloc_unit;
+    for (unsigned bin = 0; bin < free.size(); ++bin) {
+      intervals += free[bin].num_intervals();
+    }
+  }
+  ldout(cct, 30) << __func__ << " " << intervals << "/" << max_intervals 
+                 << dendl;
+  assert(intervals <= max_intervals);
+  if (!intervals || max_intervals <= 1) {
+    return 0.0;
+  }
+  intervals--;
+  max_intervals--;
+  res = (double)intervals / max_intervals;
+  return res;
+}
+
 void StupidAllocator::dump()
 {
   std::lock_guard<std::mutex> l(lock);

diff --git a/src/os/bluestore/StupidAllocator.h b/src/os/bluestore/StupidAllocator.h
index e3d86531f3f867f9e50b584eee9a6154f40e5932..a41feddc1ded3affca871f313798b427c117ce8a 100644 (file)
--- a/src/os/bluestore/StupidAllocator.h
+++ b/src/os/bluestore/StupidAllocator.h
@@ -53,6 +53,7 @@ public:
     const interval_set<uint64_t>& release_set) override;
 
   uint64_t get_free() override;
+  double get_fragmentation(uint64_t alloc_unit) override;
 
   void dump() override;
 

diff --git a/src/test/objectstore/Allocator_test.cc b/src/test/objectstore/Allocator_test.cc
index 66a5567238a4cd15c486919b3ac663fcfa817413..a53c2f7ff3a135d1b1cb6c4b91dcedf9e2aec59f 100644 (file)
--- a/src/test/objectstore/Allocator_test.cc
+++ b/src/test/objectstore/Allocator_test.cc
@@ -290,6 +290,58 @@ TEST_P(AllocTest, test_alloc_non_aligned_len)
   EXPECT_EQ(want_size, alloc->allocate(want_size, alloc_unit, 0, &extents));
 }
 
+TEST_P(AllocTest, test_alloc_fragmentation)
+{
+  if (GetParam() == std::string("bitmap")) {
+    return;
+  }
+  uint64_t capacity = 4 * 1024 * 1024;
+  uint64_t alloc_unit = 4096;
+  uint64_t want_size = alloc_unit;
+  PExtentVector allocated, tmp;
+  
+  init_alloc(capacity, alloc_unit);
+  alloc->init_add_free(0, capacity);
+  
+  EXPECT_EQ(0.0, alloc->get_fragmentation(alloc_unit));
+  for (size_t i = 0; i < capacity / alloc_unit; ++i)
+  {
+    tmp.clear();
+    EXPECT_EQ(0, alloc->reserve(want_size));
+    EXPECT_EQ(want_size, alloc->allocate(want_size, alloc_unit, 0, 0, &tmp));
+    allocated.insert(allocated.end(), tmp.begin(), tmp.end());
+    EXPECT_EQ(0.0, alloc->get_fragmentation(alloc_unit));
+  }
+  EXPECT_EQ(-ENOSPC, alloc->reserve(want_size));
+
+  for (size_t i = 0; i < allocated.size(); i += 2)
+  {
+    interval_set<uint64_t> release_set;
+    release_set.insert(allocated[i].offset, allocated[i].length);
+    alloc->release(release_set);
+  }
+  EXPECT_EQ(1.0, alloc->get_fragmentation(alloc_unit));
+  for (size_t i = 1; i < allocated.size() / 2; i += 2)
+  {
+    interval_set<uint64_t> release_set;
+    release_set.insert(allocated[i].offset, allocated[i].length);
+    alloc->release(release_set);
+  }
+  // fragmentation approx = 257 intervals / 768 max intervals
+  EXPECT_EQ(33, uint64_t(alloc->get_fragmentation(alloc_unit) * 100));
+
+  for (size_t i = allocated.size() / 2 + 1; i < allocated.size(); i += 2)
+  {
+    interval_set<uint64_t> release_set;
+    release_set.insert(allocated[i].offset, allocated[i].length);
+    alloc->release(release_set);
+  }
+  // doing some rounding trick as stupid allocator doesn't merge all the 
+  // extents that causes some minor fragmentation (minor bug or by-design behavior?).
+  // Hence leaving just two 
+  // digits after decimal point due to this.
+  EXPECT_EQ(0, uint64_t(alloc->get_fragmentation(alloc_unit) * 100));
+}
 
 INSTANTIATE_TEST_CASE_P(
   Allocator,