os/bluestore/AvlAllocator: introduce bluestore_avl_alloc_ff_max_search_bytes

so AvlAllocator can switch from the first-first mode to best-fit mode
without walking through the whole space map tree. in the
highly-fragmented system, iterating the whole tree could hurt the
performance of fast storage system a lot.

the idea comes from openzfs's metaslab allocator.

Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 05d7d38234a3303ddf06d6aa8328d89edeea6aee..1dd0b57d467f14d5319a93d61f1e07d488631fe8 100644 (file)
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -5035,6 +5035,12 @@ options:
   desc: Search for this many ranges in first-fit mode before switching over to
     to best-fit mode. 0 to iterate through all ranges for required chunk.
   default: 100
+- name: bluestore_avl_alloc_ff_max_search_bytes
+  type: size
+  level: dev
+  desc: Maximum distance to search in first-fit mode before switching over to
+    to best-fit mode. 0 to iterate through all ranges for required chunk.
+  default: 16_M
 - name: bluestore_avl_alloc_bf_threshold
   type: uint
   level: dev

diff --git a/src/os/bluestore/AvlAllocator.cc b/src/os/bluestore/AvlAllocator.cc
index 1d463b31a5546b02f238bb75781431e8a55d5b6b..e7a9befef051b0a112717a130429dbc0f9f233e8 100644 (file)
--- a/src/os/bluestore/AvlAllocator.cc
+++ b/src/os/bluestore/AvlAllocator.cc
@@ -35,6 +35,7 @@ uint64_t AvlAllocator::_pick_block_after(uint64_t *cursor,
 {
   const auto compare = range_tree.key_comp();
   uint32_t search_count = 0;
+  uint64_t search_bytes = 0;
   auto rs_start = range_tree.lower_bound(range_t{*cursor, size}, compare);
   for (auto rs = rs_start; rs != range_tree.end(); ++rs) {
     uint64_t offset = p2roundup(rs->start, align);
@@ -45,6 +46,10 @@ uint64_t AvlAllocator::_pick_block_after(uint64_t *cursor,
     if (max_search_count > 0 && ++search_count > max_search_count) {
       return -1ULL;
     }
+    if (search_bytes = rs->start - rs_start->start;
+       max_search_bytes > 0 && search_bytes > max_search_bytes) {
+      return -1ULL;
+    }
   }
   if (*cursor == 0) {
     // If we already started from beginning, don't bother with searching from beginning
@@ -60,6 +65,9 @@ uint64_t AvlAllocator::_pick_block_after(uint64_t *cursor,
     if (max_search_count > 0 && ++search_count > max_search_count) {
       return -1ULL;
     }
+    if (max_search_bytes > 0 && search_bytes + rs->start > max_search_bytes) {
+      return -1ULL;
+    }
   }
   return -1ULL;
 }
@@ -332,6 +340,8 @@ AvlAllocator::AvlAllocator(CephContext* cct,
     cct->_conf.get_val<uint64_t>("bluestore_avl_alloc_bf_free_pct")),
   max_search_count(
     cct->_conf.get_val<uint64_t>("bluestore_avl_alloc_ff_max_search_count")),
+  max_search_bytes(
+    cct->_conf.get_val<Option::size_t>("bluestore_avl_alloc_ff_max_search_bytes")),
   range_count_cap(max_mem / sizeof(range_seg_t)),
   cct(cct)
 {}

diff --git a/src/os/bluestore/AvlAllocator.h b/src/os/bluestore/AvlAllocator.h
index f813fa41c825ac822f8fb88a76daf14d8d7e74eb..3779a6702947ef9488b8cb45251ab5b5f76cf9c1 100644 (file)
--- a/src/os/bluestore/AvlAllocator.h
+++ b/src/os/bluestore/AvlAllocator.h
@@ -164,6 +164,12 @@ private:
    * becomes the performance limiting factor on high-performance storage.
    */
   const uint32_t max_search_count;
+  /*
+   * Maximum distance to search forward from the last offset, without this
+   * limit, fragmented device can see lots of iterations and _block_picker()
+   * becomes the performance limiting factor on high-performance storage.
+   */
+  const uint32_t max_search_bytes;
   /*
   * Max amount of range entries allowed. 0 - unlimited
   */