From 54cbe2f88f58c2caacb57086a35fa5016f6bd45f Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Fri, 2 Feb 2024 17:09:20 +0000 Subject: [PATCH] os/bluestore: Tests for multi bdev label feature Signed-off-by: Adam Kupczyk (cherry picked from commit b53a6d04295f4a6770f80b3ad182abdb41d84e92) --- src/test/objectstore/store_test.cc | 283 +++++++++++++++++++++ src/test/objectstore/store_test_fixture.cc | 4 + src/test/objectstore/store_test_fixture.h | 11 +- 3 files changed, 297 insertions(+), 1 deletion(-) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 75fc4835638..f3893f3aa45 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -12,6 +12,7 @@ * */ +#include #include #include #include @@ -25,6 +26,7 @@ #include #include +#include "global/global_context.h" #include "os/ObjectStore.h" #if defined(WITH_BLUESTORE) #include "os/bluestore/BlueStore.h" @@ -180,6 +182,51 @@ public: }; +class MultiLabelTest : public StoreTestDeferredSetup { + public: + std::string get_data_dir() { + return data_dir; + } + bool mounted = false; + void mount() { + ASSERT_FALSE(mounted); + store->mount(); + mounted = true; + } + void umount() { + ASSERT_TRUE(mounted); + store->umount(); + mounted = false; + } + bool corrupt_disk_at(uint64_t position) { + int fd = -1; + auto close_fd = make_scope_guard([&] { + if (fd != -1) ::close(fd); }); + string block_file = get_data_dir() + "/block"; + fd = ::open(block_file.c_str(), O_RDWR|O_CLOEXEC); + if (fd < 0) return false; + char data_fill[100] = {55}; + int r = ::pwrite(fd, data_fill, 100, position); + if (r != 100) return false; + r = ::fsync(fd); + if (r != 0) return false; + return true; + } + protected: + void DeferredSetup() { + StoreTest::SetUp(); + mounted = true; + } + void TearDown() override { + if (mounted) { + store->umount(); + } + StoreTest::RemoveTestObjectStore(); + store = nullptr; + StoreTest::TearDown(); + } +}; + class StoreTestSpecificAUSize : public StoreTestDeferredSetup { public: @@ -7058,6 +7105,11 @@ INSTANTIATE_TEST_SUITE_P( "bluestore")); #endif +INSTANTIATE_TEST_SUITE_P( + ObjectStore, + MultiLabelTest, + ::testing::Values( + "bluestore")); struct deferred_test_t { uint32_t bdev_block_size; @@ -10305,6 +10357,237 @@ TEST_P(StoreTest, mergeRegionTest) { } } +TEST_P(MultiLabelTest, MultiSelectableOff) { + SetVal(g_conf(), "bluestore_bdev_label_multi", "false"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + store->umount(); + bluestore_bdev_label_t label; + int r = BlueStore::_read_bdev_label( + g_ceph_context, get_data_dir() + "/block", &label, 0); + ASSERT_EQ(r, 0); + ASSERT_EQ(label.meta.end(), label.meta.find("multi")); + store->mount(); +} + +TEST_P(MultiLabelTest, MultiSelectableOn) { + SetVal(g_conf(), "bluestore_bdev_label_multi", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + store->umount(); + bluestore_bdev_label_t label; + int r = BlueStore::_read_bdev_label( + g_ceph_context, get_data_dir() + "/block", &label, 0); + ASSERT_EQ(r, 0); + auto it = label.meta.find("multi"); + ASSERT_NE(label.meta.end(), it); + ASSERT_EQ(it->second, "yes"); + store->mount(); +} + +TEST_P(MultiLabelTest, DetectCorruptedFirst) { + SetVal(g_conf(), "bluestore_block_size", + stringify(101L * 1024 * 1024 * 1024).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + umount(); + bool corrupt = corrupt_disk_at(0); + ASSERT_EQ(corrupt, true); + ASSERT_EQ(store->fsck(false), 1); + //store->mount(); +} + +TEST_P(MultiLabelTest, FixCorruptedFirst) { + SetVal(g_conf(), "bluestore_block_size", + stringify(101L * 1024 * 1024 * 1024).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + umount(); + bool corrupt = corrupt_disk_at(0); + ASSERT_EQ(corrupt, true); + ASSERT_EQ(store->fsck(false), 1); + ASSERT_EQ(store->repair(false), 0); + mount(); +} + +TEST_P(MultiLabelTest, FixCorruptedTwo) { + static constexpr uint64_t _1G = uint64_t(1024)*1024*1024; + SetVal(g_conf(), "bluestore_block_size", + stringify(101 * _1G).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + umount(); + bool corrupt = corrupt_disk_at(0); + ASSERT_EQ(corrupt, true); + corrupt = corrupt_disk_at(_1G); + ASSERT_EQ(corrupt, true); + ASSERT_EQ(store->fsck(false), 2); + ASSERT_EQ(store->repair(false), 0); + mount(); +} + +TEST_P(MultiLabelTest, FixCorruptedThree) { + static constexpr uint64_t _1G = uint64_t(1024)*1024*1024; + SetVal(g_conf(), "bluestore_block_size", + stringify(101 * _1G).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + umount(); + bool corrupt = corrupt_disk_at(0); + ASSERT_EQ(corrupt, true); + corrupt = corrupt_disk_at(_1G); + ASSERT_EQ(corrupt, true); + corrupt = corrupt_disk_at(_1G * 10); + ASSERT_EQ(corrupt, true); + ASSERT_EQ(store->fsck(false), 3); + ASSERT_EQ(store->repair(false), 0); + mount(); +} + +TEST_P(MultiLabelTest, CantFixCorruptedAll) { + static constexpr uint64_t _1G = uint64_t(1024)*1024*1024; + SetVal(g_conf(), "bluestore_block_size", + stringify(101 * _1G).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + umount(); + bool corrupt = corrupt_disk_at(0); + ASSERT_EQ(corrupt, true); + corrupt = corrupt_disk_at(_1G); + ASSERT_EQ(corrupt, true); + corrupt = corrupt_disk_at(_1G * 10); + ASSERT_EQ(corrupt, true); + corrupt = corrupt_disk_at(_1G * 100); + ASSERT_EQ(corrupt, true); + ASSERT_NE(store->fsck(false), 0); + ASSERT_NE(store->repair(false), 0); +} + +TEST_P(MultiLabelTest, SelectNewestLabel) { + static constexpr uint64_t _1G = uint64_t(1024)*1024*1024; + SetVal(g_conf(), "bluestore_block_size", + stringify(101 * _1G).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + umount(); + bluestore_bdev_label_t label; + int r = BlueStore::_read_bdev_label( + g_ceph_context, get_data_dir() + "/block", &label, 0); + ASSERT_EQ(r, 0); + auto it = label.meta.find("epoch"); + ASSERT_NE(it, label.meta.end()); + it->second += "1"; //APPEND "1", not add + label.meta["canary"]="alive"; + r = BlueStore::_write_bdev_label( + g_ceph_context, get_data_dir() + "/block", label, {_1G}); + ASSERT_EQ(r, 0); + ASSERT_EQ(store->fsck(false), 3); + ASSERT_EQ(store->repair(false), 0); + r = BlueStore::_read_bdev_label( + g_ceph_context, get_data_dir() + "/block", &label, 0); + ASSERT_EQ(r, 0); + ASSERT_EQ(label.meta["canary"], "alive"); +} + +TEST_P(MultiLabelTest, UpgradeToMultiLabel) { + static constexpr uint64_t _1G = uint64_t(1024)*1024*1024; + SetVal(g_conf(), "bluestore_block_size", stringify(101 * _1G).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "false"); + SetVal(g_conf(), "bluestore_bdev_label_multi_upgrade", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + umount(); + ASSERT_EQ(store->repair(false), 0); + ASSERT_EQ(store->fsck(false), 0); + bluestore_bdev_label_t label; + int r = BlueStore::_read_bdev_label( + g_ceph_context, get_data_dir() + "/block", &label, 0); + ASSERT_EQ(r, 0); + auto it = label.meta.find("epoch"); + ASSERT_NE(it, label.meta.end()); + ASSERT_EQ(label.meta["multi"], "yes"); +} + +TEST_P(MultiLabelTest, UpgradeToMultiLabelCollisionWithBlueFS) { + static constexpr uint64_t _1G = uint64_t(1024)*1024*1024; + static constexpr uint64_t _1M = uint64_t(1)*1024*1024; + SetVal(g_conf(), "bluestore_debug_inject_allocation_from_file_failure", "0"); + SetVal(g_conf(), "bluestore_block_size", stringify(101 * _1G).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "false"); + SetVal(g_conf(), "bluestore_bdev_label_multi_upgrade", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + //fill BlueFS with data + BlueStore* bstore = dynamic_cast (store.get()); + ceph_assert(bstore); + for (size_t i = 0; i < 128; i++) { + std::string name = to_string(i); + bstore->inject_bluefs_file("db", name, 16 * _1M); + } + umount(); + ASSERT_EQ(store->repair(false), 1); + ASSERT_EQ(store->fsck(false), 1); + bluestore_bdev_label_t label; + int r = BlueStore::_read_bdev_label( + g_ceph_context, get_data_dir() + "/block", &label, 0); + ASSERT_EQ(r, 0); + auto it = label.meta.find("epoch"); + ASSERT_NE(it, label.meta.end()); + ASSERT_EQ(label.meta["multi"], "yes"); +} + +TEST_P(MultiLabelTest, UpgradeToMultiLabelCollisionObjects) { + static constexpr uint64_t _1G = uint64_t(1024)*1024*1024; + static constexpr uint64_t _1M = uint64_t(1)*1024*1024; + SetVal(g_conf(), "bluestore_block_db_create", "true"); + SetVal(g_conf(), "bluestore_block_db_size", stringify(10 * _1G).c_str()); + SetVal(g_conf(), "bluestore_debug_inject_allocation_from_file_failure", "0"); + SetVal(g_conf(), "bluestore_block_size", stringify(101 * _1G).c_str()); + SetVal(g_conf(), "bluestore_bdev_label_multi", "false"); + SetVal(g_conf(), "bluestore_bdev_label_multi_upgrade", "true"); + g_conf().apply_changes(nullptr); + DeferredSetup(); + //fill with object data + coll_t cid; + auto ch = store->create_new_collection(cid); + { + ObjectStore::Transaction t; + t.create_collection(cid, 0); + int r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + bufferlist bl; + bl.append(std::string(_1M, '0')); + for (int m = 0; m < 128 + 10; m++) { + for (int n = 0; n < 16; n++) { + ObjectStore::Transaction t; + std::string name = "OBJ-"+to_string(m)+"-"+to_string(n); + ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP))); + t.write(cid, hoid, 0, bl.length(), bl); + int r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + } + } + umount(); + ASSERT_EQ(store->repair(false), 0); + ASSERT_EQ(store->fsck(false), 0); + bluestore_bdev_label_t label; + int r = BlueStore::_read_bdev_label( + g_ceph_context, get_data_dir() + "/block", &label, 0); + ASSERT_EQ(r, 0); + auto it = label.meta.find("epoch"); + ASSERT_NE(it, label.meta.end()); + ASSERT_EQ(label.meta["multi"], "yes"); +} + TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) { if (string(GetParam()) != "bluestore") return; diff --git a/src/test/objectstore/store_test_fixture.cc b/src/test/objectstore/store_test_fixture.cc index 0cffd79a709..3f9b93a3e24 100644 --- a/src/test/objectstore/store_test_fixture.cc +++ b/src/test/objectstore/store_test_fixture.cc @@ -129,3 +129,7 @@ void StoreTestFixture::CloseAndReopen() { ASSERT_EQ(0, store->mount()); g_conf().set_safe_to_start_threads(); } + +void StoreTestFixture::RemoveTestObjectStore() { + rm_r(data_dir); +} \ No newline at end of file diff --git a/src/test/objectstore/store_test_fixture.h b/src/test/objectstore/store_test_fixture.h index 0495c21bd32..afde3d3ec72 100644 --- a/src/test/objectstore/store_test_fixture.h +++ b/src/test/objectstore/store_test_fixture.h @@ -8,11 +8,13 @@ class ObjectStore; class StoreTestFixture : virtual public ::testing::Test { const std::string type; - const std::string data_dir; std::stack> saved_settings; ConfigProxy* conf = nullptr; +protected: + const std::string data_dir; + public: std::unique_ptr store; ObjectStore::CollectionHandle ch; @@ -41,4 +43,11 @@ public: } void PopSettings(size_t); void CloseAndReopen(); + void RemoveTestObjectStore(); + const std::string get_type() const { + return type; + } + const std::string get_data_dir() const { + return data_dir; + } }; -- 2.39.5