os/bluestore: make zone/span size of bitmap-allocator configurable

Perviously as zone_size_block and span_size are identical, so we could use
span_size only to simplify the calcuation.

Now that the zone_size_block and span_size are both configurable, so
it is necessary to treat them respectively.

Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 221e8507d4d537bd1f070fb1afeafc005ae4a33a..b4da25107bc1f59d6abaec0ced5fbd69433a0f31 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -972,6 +972,8 @@ OPTION(bluestore_kvbackend, OPT_STR, "rocksdb")
 OPTION(bluestore_allocator, OPT_STR, "bitmap")     // stupid | bitmap
 OPTION(bluestore_freelist_type, OPT_STR, "bitmap") // extent | bitmap
 OPTION(bluestore_freelist_blocks_per_key, OPT_INT, 128)
+OPTION(bluestore_bitmapallocator_blocks_per_zone, OPT_INT, 1024) // must be power of 2 aligned, e.g., 512, 1024, 2048...
+OPTION(bluestore_bitmapallocator_span_size, OPT_INT, 1024) // must be power of 2 aligned, e.g., 512, 1024, 2048...
 OPTION(bluestore_rocksdb_options, OPT_STR, "compression=kNoCompression,max_write_buffer_number=16,min_write_buffer_number_to_merge=3,recycle_log_file_num=16")
 OPTION(bluestore_fsck_on_mount, OPT_BOOL, false)
 OPTION(bluestore_fsck_on_umount, OPT_BOOL, false)

diff --git a/src/os/bluestore/BitAllocator.cc b/src/os/bluestore/BitAllocator.cc
index c464a715bc49630d17f268cb6924108e98a1267c..190962afc086d594e43acf3ca596d43002e94188 100644 (file)
--- a/src/os/bluestore/BitAllocator.cc
+++ b/src/os/bluestore/BitAllocator.cc
@@ -18,6 +18,7 @@
  * of the interfaces defined in BitMapArea.
  */
 
+#include "common/dout.h"
 #include "BitAllocator.h"
 #include <assert.h>
 #include <math.h>
@@ -26,7 +27,7 @@
 
 int64_t BitMapAreaLeaf::count = 0;
 int64_t BitMapZone::count = 0;
-int64_t BitMapZone::total_blocks = BITMAP_SPAN_SIZE;
+int64_t BitMapZone::total_blocks = 0;
 
 /*
  * BmapEntityList functions.
@@ -345,6 +346,7 @@ int BmapEntry::find_any_free_bits(int start_offset, int64_t num_blocks,
 void BitMapZone::init(int64_t zone_num, int64_t total_blocks, bool def)
 {
   m_area_index = zone_num;
+  BitMapZone::total_blocks = total_blocks;
   debug_assert(size() > 0);
   m_type = ZONE;
 
@@ -634,9 +636,14 @@ int64_t BitMapZone::alloc_blocks_dis(int64_t num_blocks, int64_t zone_blk_off, i
 /*
  * BitMapArea Leaf and non-Leaf functions.
  */
+int64_t BitMapArea::get_zone_size()
+{
+  return g_conf->bluestore_bitmapallocator_blocks_per_zone;
+}
+
 int64_t BitMapArea::get_span_size()
 {
-  return BITMAP_SPAN_SIZE;
+  return g_conf->bluestore_bitmapallocator_span_size;
 }
 
 bmap_area_type_t BitMapArea::level_to_type(int level)
@@ -653,8 +660,9 @@ bmap_area_type_t BitMapArea::level_to_type(int level)
 int BitMapArea::get_level(int64_t total_blocks)
 {
   int level = 1;
-  int64_t span_size = BitMapArea::get_span_size();
-  int64_t spans = span_size * span_size;
+  int64_t zone_size_block = get_zone_size();
+  int64_t span_size = get_span_size();
+  int64_t spans = zone_size_block * span_size;
   while (spans < total_blocks) {
     spans *= span_size;
     level++;
@@ -662,6 +670,24 @@ int BitMapArea::get_level(int64_t total_blocks)
   return level;
 }
 
+int64_t BitMapArea::get_level_factor(int level)
+{
+  debug_assert(level > 0);
+
+  int64_t zone_size = get_zone_size();
+  if (level == 1) {
+    return zone_size;
+  }
+
+  int64_t level_factor = zone_size;
+  int64_t span_size = get_span_size();
+  while (--level) {
+    level_factor *= span_size;
+  }
+
+  return level_factor;
+}
+
 int64_t BitMapArea::get_index()
 {
   return m_area_index;
@@ -697,7 +723,7 @@ void BitMapAreaIN::init(int64_t total_blocks, int64_t area_idx, bool def)
   debug_assert(!(total_blocks % BmapEntry::size()));
 
   init_common(total_blocks, area_idx, def);
-  int64_t level_factor = pow(BitMapArea::get_span_size(), m_level);
+  int64_t level_factor = BitMapArea::get_level_factor(m_level);
 
   num_child = (total_blocks + level_factor - 1) / level_factor;
   debug_assert(num_child < std::numeric_limits<int16_t>::max());
@@ -1054,11 +1080,15 @@ void BitMapAreaLeaf::init(int64_t total_blocks, int64_t area_idx,
   debug_assert(!(total_blocks % BmapEntry::size()));
 
   init_common(total_blocks, area_idx, def);
-  num_child = total_blocks / pow(BitMapArea::get_span_size(), m_level);
+  debug_assert(m_level == 1);
+  int zone_size_block = get_zone_size();
+  debug_assert(zone_size_block > 0);
+  num_child = (total_blocks + zone_size_block - 1) / zone_size_block;
+  debug_assert(num_child);
   m_child_size_blocks = total_blocks / num_child;
 
   debug_assert(m_level == 1);
-   BitMapArea **children = new BitMapArea*[num_child];
+  BitMapArea **children = new BitMapArea*[num_child];
   for (int i = 0; i < num_child; i++) {
       children[i] = new BitMapZone(m_child_size_blocks, i, def);
   }
@@ -1358,7 +1388,7 @@ bool BitAllocator::check_input(int64_t num_blocks)
     return false;
   }
 
-  if (num_blocks > BitMapArea::get_span_size()) {
+  if (num_blocks > get_zone_size()) {
     return false;
   }
   return true;

diff --git a/src/os/bluestore/BitAllocator.h b/src/os/bluestore/BitAllocator.h
index ec3227caea58307d32eedf35514fed5138194026..e35fc9800eb119d00132715bd69d12eafa769c46 100644 (file)
--- a/src/os/bluestore/BitAllocator.h
+++ b/src/os/bluestore/BitAllocator.h
@@ -9,7 +9,6 @@
 #define CEPH_OS_BLUESTORE_BITALLOCATOR_H
 
 #define debug_assert assert
-#define BITMAP_SPAN_SIZE (1024)
 
 #include <assert.h>
 #include <stdint.h>
@@ -187,9 +186,11 @@ protected:
   bmap_area_type_t m_type;
 
 public:
+  static int64_t get_zone_size();
   static int64_t get_span_size();
   bmap_area_type_t level_to_type(int level);
   static int get_level(int64_t total_blocks);
+  static int64_t get_level_factor(int level);
   virtual bool is_allocated(int64_t start_block, int64_t num_blocks) = 0;
   virtual bool is_exhausted() = 0;
   virtual bool child_check_n_lock(BitMapArea *child, int64_t required) {

diff --git a/src/os/bluestore/BitMapAllocator.cc b/src/os/bluestore/BitMapAllocator.cc
index 1f4b2989814ff7ded0dc55ad94c85804c7e95641..04f5dc70025c05529d302bfda5bfc7ef0f36edc8 100644 (file)
--- a/src/os/bluestore/BitMapAllocator.cc
+++ b/src/os/bluestore/BitMapAllocator.cc
@@ -23,7 +23,23 @@ BitMapAllocator::BitMapAllocator(int64_t device_size, int64_t block_size)
   : m_num_uncommitted(0),
     m_num_committing(0)
 {
-  int64_t zone_size_blks = 1024; // Change it later
+  int64_t zone_size_blks = g_conf->bluestore_bitmapallocator_blocks_per_zone;
+  assert((zone_size_blks & (zone_size_blks - 1)) == 0);
+  if (zone_size_blks & (zone_size_blks - 1)) {
+    derr << __func__ << " zone_size " << zone_size_blks
+         << " not power of 2 aligned!"
+         << dendl;
+    return;
+  }
+
+  int64_t span_size = g_conf->bluestore_bitmapallocator_span_size;
+  assert((span_size & (span_size - 1)) == 0);
+  if (span_size & (span_size - 1)) {
+    derr << __func__ << " span_size " << span_size
+         << " not power of 2 aligned!"
+         << dendl;
+    return;
+  }
 
   m_block_size = block_size;
   m_bit_alloc = new BitAllocator(device_size / block_size,

diff --git a/src/test/objectstore/BitAllocator_test.cc b/src/test/objectstore/BitAllocator_test.cc
index 190b40aebff6aa689de804bb5e4c8bf94fa91bef..44898775d389a5f76402e2392a5dc4f315de1c72 100644 (file)
--- a/src/test/objectstore/BitAllocator_test.cc
+++ b/src/test/objectstore/BitAllocator_test.cc
@@ -5,10 +5,14 @@
  * Author: Ramesh Chander, Ramesh.Chander@sandisk.com
  */
 
+#include "include/Context.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
 #include "os/bluestore/BitAllocator.h"
 #include <stdio.h>
 #include <assert.h>
 #include <math.h>
+#include <sstream>
 #include <gtest/gtest.h>
 
 #define bmap_test_assert(x) EXPECT_EQ(true, (x))
@@ -341,138 +345,161 @@ TEST(BitAllocator, test_zone_alloc)
 
 TEST(BitAllocator, test_bmap_alloc)
 {
-  int64_t total_blocks = 1024 * 4;
-  int64_t zone_size = 1024;
-  int64_t allocated = 0;
-  int64_t start_block = 0;
-
-  BitAllocator *alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT);
+  const int max_iter = 2;
+
+  for (int round = 0; round < 3; round++) {
+    // Test zone of different sizes: 512, 1024, 2048
+    int64_t zone_size = 512ull << round;
+    ostringstream val;
+    val << zone_size;
+    g_conf->set_val("bluestore_bitmapallocator_blocks_per_zone", val.str());
+
+    // choose randomized span_size
+    int64_t span_size = 512ull << (rand() % 4);
+    val.str("");
+    val << span_size;
+    g_conf->set_val("bluestore_bitmapallocator_span_size", val.str());
+    g_ceph_context->_conf->apply_changes(NULL);
+
+    int64_t total_blocks = zone_size * 4;
+    int64_t allocated = 0;
+    int64_t start_block = 0;
+
+    BitAllocator *alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT);
+
+    for (int64_t iter = 0; iter < max_iter; iter++) {
+      for (int64_t i = 0; i < total_blocks; i++) {
+        allocated = alloc->alloc_blocks(1, &start_block);
+        bmap_test_assert(allocated == 1);
+        bmap_test_assert(start_block == i);
+      }
 
-  for (int64_t iter = 0; iter < 4; iter++) {
-    for (int64_t i = 0; i < total_blocks; i++) {
-      allocated = alloc->alloc_blocks(1, &start_block);
-      bmap_test_assert(allocated == 1);
-      bmap_test_assert(start_block == i);
+      for (int64_t i = 0; i < total_blocks; i++) {
+        alloc->free_blocks(i, 1);
+      }
     }
 
-    for (int64_t i = 0; i < total_blocks; i++) {
-      alloc->free_blocks(i, 1);
-    }
-  }
+    for (int64_t iter = 0; iter < max_iter; iter++) {
+      for (int64_t i = 0; i < total_blocks / zone_size; i++) {
+        allocated = alloc->alloc_blocks(zone_size, &start_block);
+        bmap_test_assert(allocated == zone_size);
+        bmap_test_assert(start_block == i * zone_size);
+      }
 
-  for (int64_t iter = 0; iter < 4; iter++) {
-    for (int64_t i = 0; i < total_blocks / zone_size; i++) {
-      allocated = alloc->alloc_blocks(zone_size, &start_block);
-      bmap_test_assert(allocated == zone_size);
-      bmap_test_assert(start_block == i * zone_size);
+      for (int64_t i = 0; i < total_blocks / zone_size; i++) {
+        alloc->free_blocks(i * zone_size, zone_size);
+      }
     }
 
-    for (int64_t i = 0; i < total_blocks / zone_size; i++) {
-      alloc->free_blocks(i * zone_size, zone_size);
-    }
-  }
+    allocated = alloc->alloc_blocks(1, &start_block);
+    bmap_test_assert(allocated == 1);
 
-  allocated = alloc->alloc_blocks(1, &start_block);
-  bmap_test_assert(allocated == 1);
+    allocated = alloc->alloc_blocks(zone_size - 1, &start_block);
+    bmap_test_assert(allocated == zone_size - 1);
+    bmap_test_assert(start_block == 1);
 
-  allocated = alloc->alloc_blocks(zone_size - 1, &start_block);
-  bmap_test_assert(allocated == zone_size - 1);
-  bmap_test_assert(start_block == 1);
+    allocated = alloc->alloc_blocks(1, &start_block);
+    bmap_test_assert(allocated == 1);
 
-  allocated = alloc->alloc_blocks(1, &start_block);
-  bmap_test_assert(allocated == 1);
+    allocated = alloc->alloc_blocks(zone_size, &start_block);
+    bmap_test_assert(allocated == zone_size);
+    bmap_test_assert(start_block == zone_size * 2);
 
-  allocated = alloc->alloc_blocks(zone_size, &start_block);
-  bmap_test_assert(allocated == zone_size);
-  bmap_test_assert(start_block == zone_size * 2);
+    // Dis contiguous blocks allocations
+    delete alloc;
+    alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT);
 
-  // Dis contiguous blocks allocations
-  delete alloc;
-  alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT);
+    for (int64_t i = 0; i < alloc->size(); i++) {
+      allocated = alloc->alloc_blocks(1, &start_block);
+      bmap_test_assert(allocated == 1);
+    }
+    for (int i = 0; i < alloc->size(); i += 2) {
+      alloc->free_blocks(i, 1);
+    }
 
-  for (int64_t i = 0; i < alloc->size(); i++) {
-    allocated = alloc->alloc_blocks(1, &start_block);
-    bmap_test_assert(allocated == 1);
-  }
-  for (int i = 0; i < alloc->size(); i += 2) {
-    alloc->free_blocks(i, 1);
-  }
+    int64_t blocks[alloc->size() / 2];
+    memset(blocks, 0, sizeof(blocks));
+    allocated = alloc->alloc_blocks_dis(alloc->size()/2, blocks);
+    bmap_test_assert(allocated == alloc->size() / 2);
 
-  int64_t blocks[alloc->size() / 2];
-  memset(blocks, 0, sizeof(blocks));
-  allocated = alloc->alloc_blocks_dis(alloc->size()/2, blocks);
-  bmap_test_assert(allocated == alloc->size() / 2);
+    allocated = alloc->alloc_blocks_dis(1, blocks);
+    bmap_test_assert(allocated == 0);
 
-  allocated = alloc->alloc_blocks_dis(1, blocks);
-  bmap_test_assert(allocated == 0);
+    alloc->free_blocks(alloc->size()/2, 1);
+    allocated = alloc->alloc_blocks_dis(1, blocks);
 
-  alloc->free_blocks(alloc->size()/2, 1);
-  allocated = alloc->alloc_blocks_dis(1, blocks);
+    bmap_test_assert(allocated == 1);
+    bmap_test_assert(blocks[0] == alloc->size()/2);
 
-  bmap_test_assert(allocated == 1);
-  bmap_test_assert(blocks[0] == alloc->size()/2);
+    alloc->free_blocks(0, alloc->size());
+    delete alloc;
 
-  alloc->free_blocks(0, alloc->size());
-  delete alloc;
+    // unaligned zones
+    total_blocks = zone_size * 2 + 11;
+    alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT);
 
-  // unaligned zones
-  total_blocks = 1024 * 2 + 11;
-  alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT);
+    for (int64_t iter = 0; iter < max_iter; iter++) {
+      for (int64_t i = 0; i < total_blocks; i++) {
+        allocated = alloc->alloc_blocks(1, &start_block);
+        bmap_test_assert(allocated == 1);
+        bmap_test_assert(start_block == i);
+      }
 
-  for (int64_t iter = 0; iter < 4; iter++) {
-    for (int64_t i = 0; i < total_blocks; i++) {
-      allocated = alloc->alloc_blocks(1, &start_block);
-      bmap_test_assert(allocated == 1);
-      bmap_test_assert(start_block == i);
+      for (int64_t i = 0; i < total_blocks; i++) {
+        alloc->free_blocks(i, 1);
+      }
     }
+    delete alloc;
+
+    // Make three > 3 levels tree and check allocations and dealloc
+    // in a loop
+    int64_t alloc_size = 64ull << round;
+    total_blocks = BitMapArea::get_level_factor(2) * 4;
+    alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false);
+    for (int64_t iter = 0; iter < max_iter; iter++) {
+      for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
+        allocated = alloc->alloc_blocks(alloc_size, &start_block);
+        bmap_test_assert(allocated == alloc_size);
+        bmap_test_assert(start_block == i * alloc_size);
+      }
 
-    for (int64_t i = 0; i < total_blocks; i++) {
-      alloc->free_blocks(i, 1);
-    }
-  }
-  delete alloc;
-
-  // Make three > 3 levels tree and check allocations and dealloc
-  // in a loop
-  int64_t alloc_size = 16;
-  total_blocks = pow(BITMAP_SPAN_SIZE, 2) * 4;
-  alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false);
-  for (int64_t iter = 0; iter < 3; iter++) {
-    for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
-      allocated = alloc->alloc_blocks(alloc_size, &start_block);
-      bmap_test_assert(allocated == alloc_size);
-      bmap_test_assert(start_block == i * alloc_size);
+      for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
+        alloc->free_blocks(i * alloc_size, alloc_size);
+      }
     }
 
-    for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
-      alloc->free_blocks(i * alloc_size, alloc_size);
-    }
-  }
+    delete alloc;
+    alloc = new BitAllocator(1024, zone_size, CONCURRENT, true);
+
+    alloc->free_blocks(1, 1023);
+    allocated = alloc->alloc_blocks(16, &start_block);
+    bmap_test_assert(allocated == 16);
+    bmap_test_assert(start_block == 1);
+    delete alloc;
+
+    total_blocks = BitMapArea::get_level_factor(2) * 4;
+    alloc_size = 64ull << round;
+    alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false);
+    for (int64_t iter = 0; iter < max_iter; iter++) {
+      for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
+        bmap_test_assert(alloc->reserve_blocks(alloc_size));
+        allocated = alloc->alloc_blocks_res(alloc_size, &start_block);
+        bmap_test_assert(allocated == alloc_size);
+        bmap_test_assert(start_block == i * alloc_size);
+      }
 
-  delete alloc;
-  alloc = new BitAllocator(1024, zone_size, CONCURRENT, true);
-
-  alloc->free_blocks(1, 1023);
-  alloc->alloc_blocks(16, &start_block);
-  delete alloc;
-
-  total_blocks = pow(BITMAP_SPAN_SIZE, 2) * 4;
-  alloc_size = 16;
-  alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false);
-  for (int64_t iter = 0; iter < 3; iter++) {
-    for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
-      bmap_test_assert(alloc->reserve_blocks(alloc_size));
-      allocated = alloc->alloc_blocks_res(alloc_size, &start_block);
-      bmap_test_assert(allocated == alloc_size);
-      bmap_test_assert(start_block == i * alloc_size);
+      for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
+        alloc->free_blocks(i * alloc_size, alloc_size);
+      }
     }
 
-    for (int64_t i = 0; i < total_blocks / alloc_size; i++) {
-      alloc->free_blocks(i * alloc_size, alloc_size);
-    }
+    delete alloc;
   }
 
-  delete alloc;
+  // restore to typical value
+  g_conf->set_val("bluestore_bitmapallocator_blocks_per_zone", "1024");
+  g_conf->set_val("bluestore_bitmapallocator_span_size", "1024");
+  g_ceph_context->_conf->apply_changes(NULL);
 }
 
 void
@@ -560,6 +587,15 @@ TEST(BitAllocator, test_bmap_alloc_concurrent)
 
 int main(int argc, char **argv)
 {
+  vector<const char*> args;
+  argv_to_vec(argc, (const char **)argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
   ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
+  int r = RUN_ALL_TESTS();
+  g_ceph_context->put();
+  return r;
 }