store_test: Test for NCB resilence

When object is sharded and object head is missing,
NCB recovery from onodes fails with assert.
Created test CorruptedOnodesTest/Recover_TolerateMissingHeadShard
that replictes the conditions.

Test for: https://tracker.ceph.com/issues/74645

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 9075aa08c40e216f66a4bfd124c5773658c4ffb1..b9996b914b4819453a24c01cf392fd5ddb933f97 100644 (file)
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -15,6 +15,7 @@
 
 #include <fcntl.h>
 #include <glob.h>
+#include <kv/KeyValueDB.h>
 #include <stdio.h>
 #include <string.h>
 #include <iostream>
@@ -203,22 +204,41 @@ protected:
 
 #ifdef WITH_BLUESTORE
 
-class MultiLabelTest : public StoreTestDeferredSetup {
-  public:
-  std::string get_data_dir() {
-    return data_dir;
-  }
+
+class CheckedUmount: public StoreTestDeferredSetup {
+public:
   bool mounted = false;
-  int mount() {
+  virtual int mount() {
     int r = store->mount();
     if (r == 0) mounted = true;
     return r;
   }
-  void umount() {
+  virtual void umount() {
     ASSERT_TRUE(mounted);
     store->umount();
     mounted = false;
   }
+
+protected:
+  void DeferredSetup() {
+    StoreTest::SetUp();
+    mounted = true;
+  }
+  void TearDown() override {
+    if (mounted) {
+      store->umount();
+    }
+    StoreTest::RemoveTestObjectStore();
+    store = nullptr;
+    StoreTest::TearDown();
+  }
+};
+
+class MultiLabelTest : public CheckedUmount {
+  public:
+  std::string get_data_dir() {
+    return data_dir;
+  }
   bool bdev_supports_label() {
     BlueStore* bstore = dynamic_cast<BlueStore*> (store.get());
     if (!bstore) return false;
@@ -256,21 +276,38 @@ class MultiLabelTest : public StoreTestDeferredSetup {
     bdev->close();
     return r;
   }
-  protected:
-  void DeferredSetup() {
-    StoreTest::SetUp();
-    mounted = true;
+};
+
+class CorruptedOnodesTest : public CheckedUmount {
+public:
+  std::string get_data_dir() {
+    return data_dir;
   }
-  void TearDown() override {
-    if (mounted) {
-      store->umount();
-    }
-    StoreTest::RemoveTestObjectStore();
-    store = nullptr;
-    StoreTest::TearDown();
+  int mount() override {
+    int r = store->mount();
+    if (r == 0) mounted = true;
+    return r;
+  }
+  void umount() override {
+    ASSERT_TRUE(mounted);
+    store->umount();
+    mounted = false;
   }
-};
 
+  int write_object(
+    coll_t cid,
+    ObjectStore::CollectionHandle ch,
+    ghobject_t hoid,
+    size_t size)
+  {
+    ObjectStore::Transaction t;
+    bufferlist bl;
+    bl.append(std::string(size, 'x'));
+    t.write(cid, hoid, 0, bl.length(), bl);
+    int r = queue_transaction(store, ch, std::move(t));
+    return r;
+  }
+};
 #endif // WITH_BLUESTORE
 
 class StoreTestSpecificAUSize : public StoreTestDeferredSetup {
@@ -7256,6 +7293,12 @@ INSTANTIATE_TEST_SUITE_P(
   ::testing::Values(
     "bluestore"));
 
+INSTANTIATE_TEST_SUITE_P(
+  BlueStore,
+  CorruptedOnodesTest,
+  ::testing::Values("bluestore")
+);
+
 #endif // WITH_BLUESTORE
 
 struct deferred_test_t {
@@ -11365,6 +11408,77 @@ TEST_P(MultiLabelTest, UpgradeToMultiLabelCollisionWithObjects) {
   ASSERT_EQ(label.meta["multi"], "yes");
 }
 
+TEST_P(CorruptedOnodesTest, Recover_TolerateMissingHeadShard) {
+  static constexpr uint64_t _1G = uint64_t(1024)*1024*1024;
+  static constexpr uint64_t _1M = uint64_t(1)*1024*1024;
+  SetVal(g_conf(), "bluestore_debug_inject_allocation_from_file_failure", "0");
+  SetVal(g_conf(), "bluestore_block_size", stringify(101 * _1G).c_str());
+  g_conf().apply_changes(nullptr);
+  DeferredSetup();
+
+  coll_t cid(spg_t(pg_t(1,222), shard_id_t::NO_SHARD));
+  ObjectStore::CollectionHandle ch;
+
+  ghobject_t hoid1(hobject_t(sobject_t("aaaa_Object 1", CEPH_NOSNAP),"", 1, 222,""));
+  ghobject_t hoid_special(hobject_t(sobject_t("my_special_object", CEPH_NOSNAP),"", 1, 222,""));
+  ghobject_t hoid2(hobject_t(sobject_t("zzzz_Object 2", CEPH_NOSNAP),"", 1, 222,""));
+  //set hashes to have special object in the middle
+  hoid1.hobj.       set_hash(0x00000000); //0
+  hoid_special.hobj.set_hash(0x80000000); //1
+  hoid2.hobj.       set_hash(0x40000000); //2
+  int r;
+  ch = store->create_new_collection(cid);
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid, 0);
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
+
+  write_object(cid, ch, hoid1, 4 * _1M);
+  write_object(cid, ch, hoid_special, 4 * _1M);
+  write_object(cid, ch, hoid2, 4 * _1M);
+
+  ch.reset();
+  umount();
+
+  mount();
+  BlueStore* bs = dynamic_cast<BlueStore*>(store.get());
+  ceph_assert(bs);
+  KeyValueDB* pdb = bs->get_kv();
+  KeyValueDB::Iterator it = pdb->get_iterator("O");
+  it->seek_to_first();
+  while (it->valid()) {
+    if (it->key().contains("my_special_object") &&
+    it->key().ends_with("o")) {
+      //delete main key for the object
+      auto trans = pdb->get_transaction();
+      trans->rm_single_key("O", it->key());
+      pdb->submit_transaction_sync(trans);
+      break;
+    }
+    it->next();
+  }
+  it.reset();
+  umount();
+
+  SetVal(g_conf(), "bluestore_debug_inject_allocation_from_file_failure", "1");
+  g_conf().apply_changes(nullptr);
+  mount();
+  ch = store->open_collection(cid);
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, hoid1);
+    t.remove(cid, hoid_special);
+    t.remove(cid, hoid2);
+    t.remove_collection(cid);
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
+  ch.reset();
+  umount();
+}
+
 #endif // WITH_BLUESTORE
 
 TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) {