From e8d54e3c9faeddedc2890294556cd66095b83be4 Mon Sep 17 00:00:00 2001 From: Joshua Baergen Date: Thu, 9 Nov 2023 09:43:19 -0700 Subject: [PATCH] test/librbd: Add a stress test that reproduces a crash during discard journaling See the comments in DiscardWithPruneWriteOverlap for details. Signed-off-by: Joshua Baergen --- src/test/librbd/CMakeLists.txt | 3 +- src/test/librbd/journal/test_Stress.cc | 101 +++++++++++++++++++++++++ src/test/librbd/test_main.cc | 2 + 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 src/test/librbd/journal/test_Stress.cc diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index 0ae29b8bf1857..c3f0edbea5d3c 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -19,7 +19,8 @@ set(librbd_test test_Operations.cc test_Trash.cc journal/test_Entries.cc - journal/test_Replay.cc) + journal/test_Replay.cc + journal/test_Stress.cc) add_library(rbd_test STATIC ${librbd_test}) target_link_libraries(rbd_test PRIVATE rbd_test_support diff --git a/src/test/librbd/journal/test_Stress.cc b/src/test/librbd/journal/test_Stress.cc new file mode 100644 index 0000000000000..752ecf01f05ee --- /dev/null +++ b/src/test/librbd/journal/test_Stress.cc @@ -0,0 +1,101 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_fixture.h" +#include "test/librbd/test_support.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/journal/cls_journal_types.h" +#include "cls/journal/cls_journal_client.h" +#include "journal/Journaler.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/ImageWatcher.h" +#include "librbd/internal.h" +#include "librbd/Journal.h" +#include "librbd/Operations.h" +#include "librbd/api/Io.h" +#include "librbd/api/Snapshot.h" +#include "librbd/io/AioCompletion.h" +#include "librbd/io/ImageDispatchSpec.h" +#include "librbd/io/ImageRequest.h" +#include "librbd/io/ReadResult.h" +#include "librbd/journal/Types.h" + +void register_test_journal_stress() { +} + +namespace librbd { +namespace journal { + +class TestJournalStress : public TestFixture { +}; + +TEST_F(TestJournalStress, DiscardWithPruneWriteOverlap) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + // Overlap discards and writes while discard pruning is occurring. This tests + // the conditions under which https://tracker.ceph.com/issues/63422 occurred. + + // Create an image that is multiple objects so that we can force multiple + // image extents on the discard path. + CephContext* cct = reinterpret_cast(_rados.cct()); + auto object_size = 1ull << cct->_conf.get_val("rbd_default_order"); + auto image_size = 4 * object_size; + + // Write-around cache required for overlapping I/O delays. + cct->_conf.set_val_or_die("rbd_cache_writethrough_until_flush", "false"); + cct->_conf.set_val_or_die("rbd_cache_policy", "writearound"); + + auto image_name = get_temp_image_name(); + ASSERT_EQ(0, create_image_pp(m_rbd, m_ioctx, image_name, image_size)); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(image_name, &ictx)); + + std::thread write_thread( + [ictx, object_size]() { + std::string payload(object_size, '1'); + + for (auto i = 0; i < 200; i++) { + // Alternate overlaps with the two objects that the discard below + // touches. + for (auto offset = object_size; + offset < object_size * 3; + offset += object_size) { + bufferlist payload_bl; + payload_bl.append(payload); + auto aio_comp = new librbd::io::AioCompletion(); + api::Io<>::aio_write(*ictx, aio_comp, 0, payload.size(), + std::move(payload_bl), 0, true); + ASSERT_EQ(0, aio_comp->wait_for_complete()); + aio_comp->release(); + } + } + } + ); + + auto discard_exit = false; + std::thread discard_thread( + [ictx, object_size, &discard_exit]() { + while (!discard_exit) { + // We offset the discard by -4096 bytes and set discard granularity to + // 8192; this should cause two image extents to be formed in + // AbstractImageWriteRequest::send_request() on objects 1 and 2, + // overlapping with the writes above. + auto aio_comp = new librbd::io::AioCompletion(); + api::Io<>::aio_discard(*ictx, aio_comp, object_size - 4096, + 2 * object_size, 8192, true); + ASSERT_EQ(0, aio_comp->wait_for_complete()); + aio_comp->release(); + } + } + ); + + write_thread.join(); + discard_exit = true; + discard_thread.join(); +} + +} // namespace journal +} // namespace librbd diff --git a/src/test/librbd/test_main.cc b/src/test/librbd/test_main.cc index 2ff9f69dea977..82b72b1ef7e85 100644 --- a/src/test/librbd/test_main.cc +++ b/src/test/librbd/test_main.cc @@ -17,6 +17,7 @@ extern void register_test_image_watcher(); extern void register_test_internal(); extern void register_test_journal_entries(); extern void register_test_journal_replay(); +extern void register_test_journal_stress(); extern void register_test_migration(); extern void register_test_mirroring(); extern void register_test_mirroring_watcher(); @@ -37,6 +38,7 @@ int main(int argc, char **argv) register_test_internal(); register_test_journal_entries(); register_test_journal_replay(); + register_test_journal_stress(); register_test_migration(); register_test_mirroring(); register_test_mirroring_watcher(); -- 2.39.5