From: xie xingguo Date: Sun, 26 Jun 2016 04:07:53 +0000 (+0800) Subject: os/bluestore: make zone/span size of bitmap-allocator configurable X-Git-Tag: v11.0.1~783^2~4 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=b7d09ed13727cd3475d773781670db607fe3c812;p=ceph-ci.git os/bluestore: make zone/span size of bitmap-allocator configurable Perviously as zone_size_block and span_size are identical, so we could use span_size only to simplify the calcuation. Now that the zone_size_block and span_size are both configurable, so it is necessary to treat them respectively. Signed-off-by: xie xingguo --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 221e8507d4d..b4da25107bc 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -972,6 +972,8 @@ OPTION(bluestore_kvbackend, OPT_STR, "rocksdb") OPTION(bluestore_allocator, OPT_STR, "bitmap") // stupid | bitmap OPTION(bluestore_freelist_type, OPT_STR, "bitmap") // extent | bitmap OPTION(bluestore_freelist_blocks_per_key, OPT_INT, 128) +OPTION(bluestore_bitmapallocator_blocks_per_zone, OPT_INT, 1024) // must be power of 2 aligned, e.g., 512, 1024, 2048... +OPTION(bluestore_bitmapallocator_span_size, OPT_INT, 1024) // must be power of 2 aligned, e.g., 512, 1024, 2048... OPTION(bluestore_rocksdb_options, OPT_STR, "compression=kNoCompression,max_write_buffer_number=16,min_write_buffer_number_to_merge=3,recycle_log_file_num=16") OPTION(bluestore_fsck_on_mount, OPT_BOOL, false) OPTION(bluestore_fsck_on_umount, OPT_BOOL, false) diff --git a/src/os/bluestore/BitAllocator.cc b/src/os/bluestore/BitAllocator.cc index c464a715bc4..190962afc08 100644 --- a/src/os/bluestore/BitAllocator.cc +++ b/src/os/bluestore/BitAllocator.cc @@ -18,6 +18,7 @@ * of the interfaces defined in BitMapArea. */ +#include "common/dout.h" #include "BitAllocator.h" #include #include @@ -26,7 +27,7 @@ int64_t BitMapAreaLeaf::count = 0; int64_t BitMapZone::count = 0; -int64_t BitMapZone::total_blocks = BITMAP_SPAN_SIZE; +int64_t BitMapZone::total_blocks = 0; /* * BmapEntityList functions. @@ -345,6 +346,7 @@ int BmapEntry::find_any_free_bits(int start_offset, int64_t num_blocks, void BitMapZone::init(int64_t zone_num, int64_t total_blocks, bool def) { m_area_index = zone_num; + BitMapZone::total_blocks = total_blocks; debug_assert(size() > 0); m_type = ZONE; @@ -634,9 +636,14 @@ int64_t BitMapZone::alloc_blocks_dis(int64_t num_blocks, int64_t zone_blk_off, i /* * BitMapArea Leaf and non-Leaf functions. */ +int64_t BitMapArea::get_zone_size() +{ + return g_conf->bluestore_bitmapallocator_blocks_per_zone; +} + int64_t BitMapArea::get_span_size() { - return BITMAP_SPAN_SIZE; + return g_conf->bluestore_bitmapallocator_span_size; } bmap_area_type_t BitMapArea::level_to_type(int level) @@ -653,8 +660,9 @@ bmap_area_type_t BitMapArea::level_to_type(int level) int BitMapArea::get_level(int64_t total_blocks) { int level = 1; - int64_t span_size = BitMapArea::get_span_size(); - int64_t spans = span_size * span_size; + int64_t zone_size_block = get_zone_size(); + int64_t span_size = get_span_size(); + int64_t spans = zone_size_block * span_size; while (spans < total_blocks) { spans *= span_size; level++; @@ -662,6 +670,24 @@ int BitMapArea::get_level(int64_t total_blocks) return level; } +int64_t BitMapArea::get_level_factor(int level) +{ + debug_assert(level > 0); + + int64_t zone_size = get_zone_size(); + if (level == 1) { + return zone_size; + } + + int64_t level_factor = zone_size; + int64_t span_size = get_span_size(); + while (--level) { + level_factor *= span_size; + } + + return level_factor; +} + int64_t BitMapArea::get_index() { return m_area_index; @@ -697,7 +723,7 @@ void BitMapAreaIN::init(int64_t total_blocks, int64_t area_idx, bool def) debug_assert(!(total_blocks % BmapEntry::size())); init_common(total_blocks, area_idx, def); - int64_t level_factor = pow(BitMapArea::get_span_size(), m_level); + int64_t level_factor = BitMapArea::get_level_factor(m_level); num_child = (total_blocks + level_factor - 1) / level_factor; debug_assert(num_child < std::numeric_limits::max()); @@ -1054,11 +1080,15 @@ void BitMapAreaLeaf::init(int64_t total_blocks, int64_t area_idx, debug_assert(!(total_blocks % BmapEntry::size())); init_common(total_blocks, area_idx, def); - num_child = total_blocks / pow(BitMapArea::get_span_size(), m_level); + debug_assert(m_level == 1); + int zone_size_block = get_zone_size(); + debug_assert(zone_size_block > 0); + num_child = (total_blocks + zone_size_block - 1) / zone_size_block; + debug_assert(num_child); m_child_size_blocks = total_blocks / num_child; debug_assert(m_level == 1); - BitMapArea **children = new BitMapArea*[num_child]; + BitMapArea **children = new BitMapArea*[num_child]; for (int i = 0; i < num_child; i++) { children[i] = new BitMapZone(m_child_size_blocks, i, def); } @@ -1358,7 +1388,7 @@ bool BitAllocator::check_input(int64_t num_blocks) return false; } - if (num_blocks > BitMapArea::get_span_size()) { + if (num_blocks > get_zone_size()) { return false; } return true; diff --git a/src/os/bluestore/BitAllocator.h b/src/os/bluestore/BitAllocator.h index ec3227caea5..e35fc9800eb 100644 --- a/src/os/bluestore/BitAllocator.h +++ b/src/os/bluestore/BitAllocator.h @@ -9,7 +9,6 @@ #define CEPH_OS_BLUESTORE_BITALLOCATOR_H #define debug_assert assert -#define BITMAP_SPAN_SIZE (1024) #include #include @@ -187,9 +186,11 @@ protected: bmap_area_type_t m_type; public: + static int64_t get_zone_size(); static int64_t get_span_size(); bmap_area_type_t level_to_type(int level); static int get_level(int64_t total_blocks); + static int64_t get_level_factor(int level); virtual bool is_allocated(int64_t start_block, int64_t num_blocks) = 0; virtual bool is_exhausted() = 0; virtual bool child_check_n_lock(BitMapArea *child, int64_t required) { diff --git a/src/os/bluestore/BitMapAllocator.cc b/src/os/bluestore/BitMapAllocator.cc index 1f4b2989814..04f5dc70025 100644 --- a/src/os/bluestore/BitMapAllocator.cc +++ b/src/os/bluestore/BitMapAllocator.cc @@ -23,7 +23,23 @@ BitMapAllocator::BitMapAllocator(int64_t device_size, int64_t block_size) : m_num_uncommitted(0), m_num_committing(0) { - int64_t zone_size_blks = 1024; // Change it later + int64_t zone_size_blks = g_conf->bluestore_bitmapallocator_blocks_per_zone; + assert((zone_size_blks & (zone_size_blks - 1)) == 0); + if (zone_size_blks & (zone_size_blks - 1)) { + derr << __func__ << " zone_size " << zone_size_blks + << " not power of 2 aligned!" + << dendl; + return; + } + + int64_t span_size = g_conf->bluestore_bitmapallocator_span_size; + assert((span_size & (span_size - 1)) == 0); + if (span_size & (span_size - 1)) { + derr << __func__ << " span_size " << span_size + << " not power of 2 aligned!" + << dendl; + return; + } m_block_size = block_size; m_bit_alloc = new BitAllocator(device_size / block_size, diff --git a/src/test/objectstore/BitAllocator_test.cc b/src/test/objectstore/BitAllocator_test.cc index 190b40aebff..44898775d38 100644 --- a/src/test/objectstore/BitAllocator_test.cc +++ b/src/test/objectstore/BitAllocator_test.cc @@ -5,10 +5,14 @@ * Author: Ramesh Chander, Ramesh.Chander@sandisk.com */ +#include "include/Context.h" +#include "common/ceph_argparse.h" +#include "global/global_init.h" #include "os/bluestore/BitAllocator.h" #include #include #include +#include #include #define bmap_test_assert(x) EXPECT_EQ(true, (x)) @@ -341,138 +345,161 @@ TEST(BitAllocator, test_zone_alloc) TEST(BitAllocator, test_bmap_alloc) { - int64_t total_blocks = 1024 * 4; - int64_t zone_size = 1024; - int64_t allocated = 0; - int64_t start_block = 0; - - BitAllocator *alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT); + const int max_iter = 2; + + for (int round = 0; round < 3; round++) { + // Test zone of different sizes: 512, 1024, 2048 + int64_t zone_size = 512ull << round; + ostringstream val; + val << zone_size; + g_conf->set_val("bluestore_bitmapallocator_blocks_per_zone", val.str()); + + // choose randomized span_size + int64_t span_size = 512ull << (rand() % 4); + val.str(""); + val << span_size; + g_conf->set_val("bluestore_bitmapallocator_span_size", val.str()); + g_ceph_context->_conf->apply_changes(NULL); + + int64_t total_blocks = zone_size * 4; + int64_t allocated = 0; + int64_t start_block = 0; + + BitAllocator *alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT); + + for (int64_t iter = 0; iter < max_iter; iter++) { + for (int64_t i = 0; i < total_blocks; i++) { + allocated = alloc->alloc_blocks(1, &start_block); + bmap_test_assert(allocated == 1); + bmap_test_assert(start_block == i); + } - for (int64_t iter = 0; iter < 4; iter++) { - for (int64_t i = 0; i < total_blocks; i++) { - allocated = alloc->alloc_blocks(1, &start_block); - bmap_test_assert(allocated == 1); - bmap_test_assert(start_block == i); + for (int64_t i = 0; i < total_blocks; i++) { + alloc->free_blocks(i, 1); + } } - for (int64_t i = 0; i < total_blocks; i++) { - alloc->free_blocks(i, 1); - } - } + for (int64_t iter = 0; iter < max_iter; iter++) { + for (int64_t i = 0; i < total_blocks / zone_size; i++) { + allocated = alloc->alloc_blocks(zone_size, &start_block); + bmap_test_assert(allocated == zone_size); + bmap_test_assert(start_block == i * zone_size); + } - for (int64_t iter = 0; iter < 4; iter++) { - for (int64_t i = 0; i < total_blocks / zone_size; i++) { - allocated = alloc->alloc_blocks(zone_size, &start_block); - bmap_test_assert(allocated == zone_size); - bmap_test_assert(start_block == i * zone_size); + for (int64_t i = 0; i < total_blocks / zone_size; i++) { + alloc->free_blocks(i * zone_size, zone_size); + } } - for (int64_t i = 0; i < total_blocks / zone_size; i++) { - alloc->free_blocks(i * zone_size, zone_size); - } - } + allocated = alloc->alloc_blocks(1, &start_block); + bmap_test_assert(allocated == 1); - allocated = alloc->alloc_blocks(1, &start_block); - bmap_test_assert(allocated == 1); + allocated = alloc->alloc_blocks(zone_size - 1, &start_block); + bmap_test_assert(allocated == zone_size - 1); + bmap_test_assert(start_block == 1); - allocated = alloc->alloc_blocks(zone_size - 1, &start_block); - bmap_test_assert(allocated == zone_size - 1); - bmap_test_assert(start_block == 1); + allocated = alloc->alloc_blocks(1, &start_block); + bmap_test_assert(allocated == 1); - allocated = alloc->alloc_blocks(1, &start_block); - bmap_test_assert(allocated == 1); + allocated = alloc->alloc_blocks(zone_size, &start_block); + bmap_test_assert(allocated == zone_size); + bmap_test_assert(start_block == zone_size * 2); - allocated = alloc->alloc_blocks(zone_size, &start_block); - bmap_test_assert(allocated == zone_size); - bmap_test_assert(start_block == zone_size * 2); + // Dis contiguous blocks allocations + delete alloc; + alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT); - // Dis contiguous blocks allocations - delete alloc; - alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT); + for (int64_t i = 0; i < alloc->size(); i++) { + allocated = alloc->alloc_blocks(1, &start_block); + bmap_test_assert(allocated == 1); + } + for (int i = 0; i < alloc->size(); i += 2) { + alloc->free_blocks(i, 1); + } - for (int64_t i = 0; i < alloc->size(); i++) { - allocated = alloc->alloc_blocks(1, &start_block); - bmap_test_assert(allocated == 1); - } - for (int i = 0; i < alloc->size(); i += 2) { - alloc->free_blocks(i, 1); - } + int64_t blocks[alloc->size() / 2]; + memset(blocks, 0, sizeof(blocks)); + allocated = alloc->alloc_blocks_dis(alloc->size()/2, blocks); + bmap_test_assert(allocated == alloc->size() / 2); - int64_t blocks[alloc->size() / 2]; - memset(blocks, 0, sizeof(blocks)); - allocated = alloc->alloc_blocks_dis(alloc->size()/2, blocks); - bmap_test_assert(allocated == alloc->size() / 2); + allocated = alloc->alloc_blocks_dis(1, blocks); + bmap_test_assert(allocated == 0); - allocated = alloc->alloc_blocks_dis(1, blocks); - bmap_test_assert(allocated == 0); + alloc->free_blocks(alloc->size()/2, 1); + allocated = alloc->alloc_blocks_dis(1, blocks); - alloc->free_blocks(alloc->size()/2, 1); - allocated = alloc->alloc_blocks_dis(1, blocks); + bmap_test_assert(allocated == 1); + bmap_test_assert(blocks[0] == alloc->size()/2); - bmap_test_assert(allocated == 1); - bmap_test_assert(blocks[0] == alloc->size()/2); + alloc->free_blocks(0, alloc->size()); + delete alloc; - alloc->free_blocks(0, alloc->size()); - delete alloc; + // unaligned zones + total_blocks = zone_size * 2 + 11; + alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT); - // unaligned zones - total_blocks = 1024 * 2 + 11; - alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT); + for (int64_t iter = 0; iter < max_iter; iter++) { + for (int64_t i = 0; i < total_blocks; i++) { + allocated = alloc->alloc_blocks(1, &start_block); + bmap_test_assert(allocated == 1); + bmap_test_assert(start_block == i); + } - for (int64_t iter = 0; iter < 4; iter++) { - for (int64_t i = 0; i < total_blocks; i++) { - allocated = alloc->alloc_blocks(1, &start_block); - bmap_test_assert(allocated == 1); - bmap_test_assert(start_block == i); + for (int64_t i = 0; i < total_blocks; i++) { + alloc->free_blocks(i, 1); + } } + delete alloc; + + // Make three > 3 levels tree and check allocations and dealloc + // in a loop + int64_t alloc_size = 64ull << round; + total_blocks = BitMapArea::get_level_factor(2) * 4; + alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false); + for (int64_t iter = 0; iter < max_iter; iter++) { + for (int64_t i = 0; i < total_blocks / alloc_size; i++) { + allocated = alloc->alloc_blocks(alloc_size, &start_block); + bmap_test_assert(allocated == alloc_size); + bmap_test_assert(start_block == i * alloc_size); + } - for (int64_t i = 0; i < total_blocks; i++) { - alloc->free_blocks(i, 1); - } - } - delete alloc; - - // Make three > 3 levels tree and check allocations and dealloc - // in a loop - int64_t alloc_size = 16; - total_blocks = pow(BITMAP_SPAN_SIZE, 2) * 4; - alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false); - for (int64_t iter = 0; iter < 3; iter++) { - for (int64_t i = 0; i < total_blocks / alloc_size; i++) { - allocated = alloc->alloc_blocks(alloc_size, &start_block); - bmap_test_assert(allocated == alloc_size); - bmap_test_assert(start_block == i * alloc_size); + for (int64_t i = 0; i < total_blocks / alloc_size; i++) { + alloc->free_blocks(i * alloc_size, alloc_size); + } } - for (int64_t i = 0; i < total_blocks / alloc_size; i++) { - alloc->free_blocks(i * alloc_size, alloc_size); - } - } + delete alloc; + alloc = new BitAllocator(1024, zone_size, CONCURRENT, true); + + alloc->free_blocks(1, 1023); + allocated = alloc->alloc_blocks(16, &start_block); + bmap_test_assert(allocated == 16); + bmap_test_assert(start_block == 1); + delete alloc; + + total_blocks = BitMapArea::get_level_factor(2) * 4; + alloc_size = 64ull << round; + alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false); + for (int64_t iter = 0; iter < max_iter; iter++) { + for (int64_t i = 0; i < total_blocks / alloc_size; i++) { + bmap_test_assert(alloc->reserve_blocks(alloc_size)); + allocated = alloc->alloc_blocks_res(alloc_size, &start_block); + bmap_test_assert(allocated == alloc_size); + bmap_test_assert(start_block == i * alloc_size); + } - delete alloc; - alloc = new BitAllocator(1024, zone_size, CONCURRENT, true); - - alloc->free_blocks(1, 1023); - alloc->alloc_blocks(16, &start_block); - delete alloc; - - total_blocks = pow(BITMAP_SPAN_SIZE, 2) * 4; - alloc_size = 16; - alloc = new BitAllocator(total_blocks, zone_size, CONCURRENT, false); - for (int64_t iter = 0; iter < 3; iter++) { - for (int64_t i = 0; i < total_blocks / alloc_size; i++) { - bmap_test_assert(alloc->reserve_blocks(alloc_size)); - allocated = alloc->alloc_blocks_res(alloc_size, &start_block); - bmap_test_assert(allocated == alloc_size); - bmap_test_assert(start_block == i * alloc_size); + for (int64_t i = 0; i < total_blocks / alloc_size; i++) { + alloc->free_blocks(i * alloc_size, alloc_size); + } } - for (int64_t i = 0; i < total_blocks / alloc_size; i++) { - alloc->free_blocks(i * alloc_size, alloc_size); - } + delete alloc; } - delete alloc; + // restore to typical value + g_conf->set_val("bluestore_bitmapallocator_blocks_per_zone", "1024"); + g_conf->set_val("bluestore_bitmapallocator_span_size", "1024"); + g_ceph_context->_conf->apply_changes(NULL); } void @@ -560,6 +587,15 @@ TEST(BitAllocator, test_bmap_alloc_concurrent) int main(int argc, char **argv) { + vector args; + argv_to_vec(argc, (const char **)argv, args); + env_to_vec(args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + int r = RUN_ALL_TESTS(); + g_ceph_context->put(); + return r; }