test/store_test: Expose race in BlueFS truncate / remove

Created test that exposes race between BlueFS::truncate and BlueFS::unlink.
Test requires injection of 1ms sleep to BlueFS::truncate.
Therefore, in this form, it is unsuitable for merge.

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
(cherry picked from commit d7f0b6a23d6e9d6bd5c760bbf94b509d50e80f7e)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index b9a57cb714c51f14b2038eda73c04687c24a5f8d..90f336e1ff8c2258bb238d04ba5fc2e7d55ebb75 100644 (file)
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -3744,7 +3744,7 @@ int BlueFS::truncate(FileWriter *h, uint64_t offset)/*_WF_L*/
   if (offset > fnode.size) {
     ceph_abort_msg("truncate up not supported");
   }
-
+  unittest_inject_delay();
   _flush_bdev(h);
   {
     std::lock_guard ll(log.lock);

diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index af66d9fdcbe9d91c3acf0a9c624c429daabc8e7e..079c880cca72a7cf78a7d99f5a62c9abe5974066 100644 (file)
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -237,6 +237,7 @@ public:
   : m_func(func) {}
   template<typename... Arg>
   void operator()(Arg... arg) { if (m_func) m_func(std::forward<Arg...>(arg...)); }
+  void operator()() { if (m_func) m_func(); }
   void operator=(T&& func) { m_func = std::move(func);}
   void operator=(T& func) { m_func = func;}
 private:
@@ -808,6 +809,7 @@ public:
   }
   uint64_t debug_get_dirty_seq(FileWriter *h);
   bool debug_get_is_dev_dirty(FileWriter *h, uint8_t dev);
+  debug_point_t<std::function<void()>> unittest_inject_delay;
 
 private:
   // Wrappers for BlockDevice::read(...) and BlockDevice::read_random(...)

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 35aa29e63c7fada6df1ceda15d3b6ace192a2457..e98e66b15825b6a0a22aa60cc045991a390ca359 100644 (file)
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -11732,6 +11732,53 @@ TEST_P(StoreTestOmapUpgrade, LargeLegacyToPG) {
   }
 }
 
+
+TEST_P(StoreTest, BlueFS_truncate_remove_race) {
+  if (string(GetParam()) != "bluestore")
+    GTEST_SKIP();
+
+  BlueStore* bstore = dynamic_cast<BlueStore*> (store.get());
+  ceph_assert(bstore);
+  BlueFS& fs = *bstore->get_bluefs();
+  fs.unittest_inject_delay = []() { usleep(1000); };
+  static constexpr uint32_t batch_size = 20;
+  std::binary_semaphore go_remove(0);
+  std::binary_semaphore done_remove(0);
+
+  auto files_remover = [&]() {
+    for (uint32_t cnt = 0; cnt < batch_size; cnt++) {
+      go_remove.acquire();
+      std::string name = "test-file-" + std::to_string(cnt);
+      fs.unlink("dir", name);
+      fs.sync_metadata(false);
+      done_remove.release();
+    }
+  };
+
+  ASSERT_EQ(0, fs.mkdir("dir"));
+  std::thread remover_thread(files_remover);
+
+  for (uint32_t cnt = 0; cnt < batch_size; cnt++) {
+    std::string name = "test-file-" + std::to_string(cnt);
+    BlueFS::FileWriter *f = nullptr;
+    ASSERT_EQ(0, fs.open_for_write("dir", name, &f, false));
+    fs.preallocate(f->file, 0, 100000);
+    for (uint32_t i = 0; i < 10; i++) {
+      fs.append_try_flush(f, "x", 1);
+      fs.fsync(f);
+    }
+    go_remove.release();
+    fs.truncate(f, 10);
+    fs.close_writer(f);
+    done_remove.acquire();
+  }
+
+  remover_thread.join();
+  fs.unittest_inject_delay = nullptr;
+  EXPECT_EQ(store->umount(), 0);
+  EXPECT_EQ(store->mount(), 0);
+}
+
 #endif  // WITH_BLUESTORE
 
 int main(int argc, char **argv) {