From a0194583ad9bb290138163af30823aba588dfc34 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Wed, 13 Dec 2023 19:05:45 +0300 Subject: [PATCH] os/bluestore: introduce bluestore_debug_enforce_min_alloc_size config parameter. This allows to override persistent min_alloc_size if needed. This might be helpful to troubleshoot and work around issues like https://tracker.ceph.com/issues/63618 Signed-off-by: Igor Fedotov --- src/common/options/global.yaml.in | 15 +++++++++ src/os/bluestore/BitmapFreelistManager.cc | 16 +++++++++ src/os/bluestore/BitmapFreelistManager.h | 2 ++ src/os/bluestore/BlueStore.cc | 40 +++++++++++++++-------- src/os/bluestore/FreelistManager.h | 2 ++ 5 files changed, 62 insertions(+), 13 deletions(-) diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index ead0d0caedd33..8d506a4f6a3fd 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -4406,6 +4406,21 @@ options: flags: - create with_legacy: true +- name: bluestore_debug_enforce_min_alloc_size + type: uint + level: dev + desc: Enforces specific min_alloc size usages + long_desc: This overrides actual min_alloc_size value persisted on mkfs + (and originally obtained from bluestore_min_alloc_size) and permits to + use arbitrary value for this value. Intended primarily for dev/debug + purposes and should be used with care and deep understanding of potential + consequences, e.g. data corruption. + default: 0 + see_also: + - bluestore_min_alloc_size + flags: + - startup + with_legacy: true - name: bluestore_use_optimal_io_size_for_min_alloc_size type: bool level: advanced diff --git a/src/os/bluestore/BitmapFreelistManager.cc b/src/os/bluestore/BitmapFreelistManager.cc index f1f4831d56711..cb2ecd689e17f 100644 --- a/src/os/bluestore/BitmapFreelistManager.cc +++ b/src/os/bluestore/BitmapFreelistManager.cc @@ -610,3 +610,19 @@ void BitmapFreelistManager::get_meta( res->emplace_back("bfm_bytes_per_block", stringify(bytes_per_block)); res->emplace_back("bfm_blocks_per_key", stringify(blocks_per_key)); } + +bool BitmapFreelistManager::validate(uint64_t min_alloc_size) const +{ + bool ret = true; + auto my_alloc_size = get_alloc_size(); + ceph_assert(my_alloc_size); + ceph_assert(min_alloc_size); + if (!is_null_manager() && + ((min_alloc_size < my_alloc_size) || (min_alloc_size % my_alloc_size))) { + derr << __func__ << " inconsistent alloc units:" << std::hex + << "0x" << get_alloc_size() << " vs. 0x" << min_alloc_size + << std::dec << dendl; + ret = false; + } + return ret; +} diff --git a/src/os/bluestore/BitmapFreelistManager.h b/src/os/bluestore/BitmapFreelistManager.h index 5b04e8fd28cc0..30389056aa75d 100644 --- a/src/os/bluestore/BitmapFreelistManager.h +++ b/src/os/bluestore/BitmapFreelistManager.h @@ -94,6 +94,8 @@ public: } void get_meta(uint64_t target_size, std::vector>*) const override; + + bool validate(uint64_t min_alloc_size) const override; }; #endif diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 7d02b1551e0f1..afd28dc35ec4c 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -6791,6 +6791,14 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t, return r; } } + dout(1) << __func__ << " effective freelist_type = " << freelist_type << std::hex + << ", freelist_alloc_size = 0x" << fm->get_alloc_size() + << ", min_alloc_size = 0x" << min_alloc_size + << std::dec << dendl; + if (!fm->validate(min_alloc_size)) { + derr << __func__ << " freelist validation failed, unable to proceed." << dendl; + ceph_assert(false); + } // if space size tracked by free list manager is that higher than actual // dev size one can hit out-of-space allocation which will result // in data loss and/or assertions @@ -13116,20 +13124,26 @@ int BlueStore::_open_super_meta() } { - bufferlist bl; - db->get(PREFIX_SUPER, "min_alloc_size", &bl); - auto p = bl.cbegin(); - try { - uint64_t val; - decode(val, p); - min_alloc_size = val; - min_alloc_size_order = std::countr_zero(val); - min_alloc_size_mask = min_alloc_size - 1; + if(cct->_conf->bluestore_debug_enforce_min_alloc_size == 0) { + bufferlist bl; + db->get(PREFIX_SUPER, "min_alloc_size", &bl); + auto p = bl.cbegin(); + try { + uint64_t val; + decode(val, p); + min_alloc_size = val; + min_alloc_size_order = std::countr_zero(val); + min_alloc_size_mask = min_alloc_size - 1; - ceph_assert(min_alloc_size == 1u << min_alloc_size_order); - } catch (ceph::buffer::error& e) { - derr << __func__ << " unable to read min_alloc_size" << dendl; - return -EIO; + ceph_assert(min_alloc_size == 1u << min_alloc_size_order); + } catch (ceph::buffer::error& e) { + derr << __func__ << " unable to read min_alloc_size" << dendl; + return -EIO; + } + } else { + min_alloc_size = cct->_conf->bluestore_debug_enforce_min_alloc_size; + min_alloc_size_order = std::countr_zero(min_alloc_size); + min_alloc_size_mask = min_alloc_size - 1; } dout(1) << __func__ << " min_alloc_size 0x" << std::hex << min_alloc_size << std::dec << dendl; diff --git a/src/os/bluestore/FreelistManager.h b/src/os/bluestore/FreelistManager.h index b647e55c2c16a..7827a47ddd133 100644 --- a/src/os/bluestore/FreelistManager.h +++ b/src/os/bluestore/FreelistManager.h @@ -52,6 +52,8 @@ public: virtual void get_meta(uint64_t target_size, std::vector>*) const = 0; + virtual bool validate(uint64_t min_alloc_size) const = 0; + void set_null_manager() { null_manager = true; } -- 2.39.5