tools/ceph-bluestore-tool: Fix bluefs-bdev-expand command

Update allocation file when we expand-device
Add the expended space to the allocator and then force an update to the allocation file

There is also a new standalone test case for expand

Fixes: https://tracker.ceph.com/issues/53699
Signed-off-by: Gabriel Benhanokh <gbenhano@redhat.com>

diff --git a/qa/standalone/osd/osd-bluefs-volume-ops.sh b/qa/standalone/osd/osd-bluefs-volume-ops.sh
index c0235ef6772e5ab0be1319bce628c8df97369a8b..aedfbc9b5cbbec88e5417c9d690563223da5fcc3 100755 (executable)
--- a/qa/standalone/osd/osd-bluefs-volume-ops.sh
+++ b/qa/standalone/osd/osd-bluefs-volume-ops.sh
@@ -400,6 +400,96 @@ function TEST_bluestore2() {
     wait_for_clean || return 1
 }
 
+function TEST_bluestore_expand() {
+    local dir=$1
+
+    local flimit=$(ulimit -n)
+    if [ $flimit -lt 1536 ]; then
+        echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens."
+    fi
+    export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one
+    export CEPH_ARGS
+    CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
+    CEPH_ARGS+="--mon-host=$CEPH_MON "
+    CEPH_ARGS+="--bluestore_block_size=4294967296 "
+    CEPH_ARGS+="--bluestore_block_db_create=true "
+    CEPH_ARGS+="--bluestore_block_db_size=1073741824 "
+    CEPH_ARGS+="--bluestore_block_wal_create=false "
+    CEPH_ARGS+="--bluestore_fsck_on_mount=true "
+    CEPH_ARGS+="--osd_pool_default_size=1 "
+    CEPH_ARGS+="--osd_pool_default_min_size=1 "
+    CEPH_ARGS+="--bluestore_debug_enforce_settings=ssd "
+
+    run_mon $dir a || return 1
+    run_mgr $dir x || return 1
+    run_osd $dir 0 || return 1
+    osd_pid0=$(cat $dir/osd.0.pid)
+
+    sleep 5
+    create_pool foo 16
+
+    # write some objects
+    timeout 60 rados bench -p foo 30 write -b 4096 --no-cleanup #|| return 1
+    sleep 5
+    
+    total_space_before=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.slow_total_bytes" )
+    free_space_before=`ceph tell osd.0 bluestore bluefs device info | grep "BDEV_SLOW" -A 2 | grep free | cut -d':' -f 2 | cut -d"," -f 1 | cut -d' ' -f 2`
+    
+    # kill
+    while kill $osd_pid0; do sleep 1 ; done
+    ceph osd down 0
+
+    # destage allocation to file before expand (in case fast-shutdown skipped that step)
+    ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 allocmap || return 1
+
+    # expand slow devices
+    ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 fsck || return 1
+
+    requested_space=4294967296 # 4GB
+    truncate $dir/0/block -s $requested_space
+    ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 bluefs-bdev-expand || return 1
+
+    # slow, DB, WAL -> slow, DB
+    ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 fsck || return 1
+
+    # compare allocation-file with RocksDB state
+    ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 qfsck || return 1
+
+    ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 bluefs-bdev-sizes
+    
+    activate_osd $dir 0 || return 1
+    osd_pid0=$(cat $dir/osd.0.pid)
+
+    wait_for_clean || return 1
+    
+    total_space_after=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.slow_total_bytes" )
+    free_space_after=`ceph tell osd.0 bluestore bluefs device info | grep "BDEV_SLOW" -A 2 | grep free | cut -d':' -f 2 | cut -d"," -f 1 | cut -d' ' -f 2`
+
+    if [$total_space_after != $requested_space]; then
+       echo "total_space_after = $total_space_after"
+       echo "requested_space   = $requested_space"
+       return 1;
+    fi
+
+    total_space_added=$((total_space_after - total_space_before))
+    free_space_added=$((free_space_after - free_space_before))
+
+    let new_used_space=($total_space_added - $free_space_added)
+    echo $new_used_space
+    # allow upto 128KB to be consumed
+    if [ $new_used_space -gt 131072 ]; then
+       echo "total_space_added = $total_space_added"
+       echo "free_space_added  = $free_space_added"
+       return 1;
+    fi
+    
+    # kill
+    while kill $osd_pid0; do sleep 1 ; done
+    ceph osd down 0
+
+    ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 qfsck || return 1
+}
+
 main osd-bluefs-volume-ops "$@"
 
 # Local Variables:

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 89cc0713a44eb57f98c46742845efc0634a579d2..7e5776cc4b06b1cb324f5677e9fbf850a32b49dc 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -6600,7 +6600,8 @@ void BlueStore::_close_db()
   dout(10) << __func__ << ":read_only=" << db_was_opened_read_only << " fm=" << fm << " destage_alloc_file=" << need_to_destage_allocation_file << dendl;
   _close_db_leave_bluefs();
 
-  if (fm && fm->is_null_manager() && !db_was_opened_read_only && need_to_destage_allocation_file) {
+  if (need_to_destage_allocation_file) {
+    ceph_assert(fm && fm->is_null_manager());
     int ret = store_allocator(alloc);
     if (ret != 0) {
       derr << __func__ << "::NCB::store_allocator() failed (continue with bitmapFreelistManager)" << dendl;
@@ -7449,6 +7450,11 @@ int BlueStore::expand_devices(ostream& out)
           << std::endl;
       }
     }
+
+    // we grow the allocation range, must reflect it in the allocation file
+    alloc->init_add_free(size0, size - size0);
+    need_to_destage_allocation_file = true;
+
     _close_db_and_around();
 
     // mount in read/write to sync expansion changes

diff --git a/src/os/bluestore/bluestore_tool.cc b/src/os/bluestore/bluestore_tool.cc
index 6cf5d3dd1a44e32f0fa3c6ea04856b8a764f925f..2f65042a050d416509a514481771d96c8de5ffb6 100644 (file)
--- a/src/os/bluestore/bluestore_tool.cc
+++ b/src/os/bluestore/bluestore_tool.cc
@@ -565,7 +565,7 @@ int main(int argc, char **argv)
 #endif
   }
   else if (action == "allocmap") {
-#ifndef CEPH_BLUESTORE_TOOL_ENABLE_ALLOCMAP
+#ifdef CEPH_BLUESTORE_TOOL_DISABLE_ALLOCMAP
     cerr << action << " bluestore.allocmap is not supported!!! " << std::endl;
     exit(EXIT_FAILURE);
 #else