From eb6278e441c7a2d652fc78ee25ff26fa8ec5f26b Mon Sep 17 00:00:00 2001 From: Connor Fawcett Date: Wed, 23 Apr 2025 15:45:24 +0100 Subject: [PATCH] Add a new flag to the IO exerciser which enables live consistency checking during IO sequences (disabled by default). Signed-off-by: Connor Fawcett --- src/common/io_exerciser/EcIoSequence.cc | 23 ++-- src/common/io_exerciser/EcIoSequence.h | 11 +- src/common/io_exerciser/IoOp.cc | 11 ++ src/common/io_exerciser/IoOp.h | 7 + src/common/io_exerciser/IoSequence.cc | 129 +++++++++++------- src/common/io_exerciser/IoSequence.h | 37 ++--- src/common/io_exerciser/OpType.h | 3 + src/common/io_exerciser/RadosIo.cc | 14 +- src/common/io_exerciser/RadosIo.h | 2 + src/test/osd/CMakeLists.txt | 2 +- .../ceph_test_rados_io_sequence.cc | 24 ++-- .../ceph_test_rados_io_sequence.h | 5 +- 12 files changed, 178 insertions(+), 90 deletions(-) diff --git a/src/common/io_exerciser/EcIoSequence.cc b/src/common/io_exerciser/EcIoSequence.cc index 1878f2f7002f8..2c61a7b381c15 100644 --- a/src/common/io_exerciser/EcIoSequence.cc +++ b/src/common/io_exerciser/EcIoSequence.cc @@ -15,7 +15,8 @@ std::unique_ptr EcIoSequence::generate_sequence( Sequence sequence, std::pair obj_size_range, std::optional> km, std::optional> mappinglayers, - int seed) { + int seed, + bool check_consistency) { switch (sequence) { case Sequence::SEQUENCE_SEQ0: [[fallthrough]]; @@ -45,16 +46,16 @@ std::unique_ptr EcIoSequence::generate_sequence( [[fallthrough]]; case Sequence::SEQUENCE_SEQ14: return std::make_unique(obj_size_range, seed, - sequence, km, mappinglayers); + sequence, km, mappinglayers, check_consistency); case Sequence::SEQUENCE_SEQ10: - return std::make_unique(obj_size_range, seed, km, mappinglayers); + return std::make_unique(obj_size_range, seed, km, mappinglayers, check_consistency); default: ceph_abort_msg("Unrecognised sequence"); } } -EcIoSequence::EcIoSequence(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), +EcIoSequence::EcIoSequence(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), setup_inject(false), clear_inject(false), shard_to_inject(std::nullopt) {} @@ -172,9 +173,10 @@ ceph::io_exerciser::ReadInjectSequence::ReadInjectSequence( int seed, Sequence s, std::optional> km, - std::optional> mappinglayers) - : EcIoSequence(obj_size_range, seed) { - child_sequence = IoSequence::generate_sequence(s, obj_size_range, seed); + std::optional> mappinglayers, + bool check_consistency) + : EcIoSequence(obj_size_range, seed, check_consistency) { + child_sequence = IoSequence::generate_sequence(s, obj_size_range, seed, check_consistency); select_random_data_shard_to_inject_read_error(km, mappinglayers); generate_random_read_inject_type(); } @@ -267,8 +269,9 @@ ceph::io_exerciser::ReadInjectSequence::_next() { ceph::io_exerciser::Seq10::Seq10( std::pair obj_size_range, int seed, std::optional> km, - std::optional> mappinglayers) - : EcIoSequence(obj_size_range, seed), + std::optional> mappinglayers, + bool check_consistency) + : EcIoSequence(obj_size_range, seed, check_consistency), offset(0), length(1), inject_error_done(false), diff --git a/src/common/io_exerciser/EcIoSequence.h b/src/common/io_exerciser/EcIoSequence.h index 8082b2267a78e..90f79da7315ec 100644 --- a/src/common/io_exerciser/EcIoSequence.h +++ b/src/common/io_exerciser/EcIoSequence.h @@ -12,7 +12,8 @@ class EcIoSequence : public IoSequence { std::optional> km, std::optional> mappinglayers, - int seed); + int seed, + bool check_consistency); protected: bool setup_inject; @@ -20,7 +21,7 @@ class EcIoSequence : public IoSequence { std::optional shard_to_inject; InjectOpType inject_op_type; - EcIoSequence(std::pair obj_size_range, int seed); + EcIoSequence(std::pair obj_size_range, int seed, bool check_consistency); // Writes cannot be sent to injected on shard zero, so selections seperated // out @@ -50,7 +51,8 @@ class ReadInjectSequence : public EcIoSequence { std::pair obj_size_range, int seed, Sequence s, std::optional> km, std::optional> - mappinglayers); + mappinglayers, + bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -67,7 +69,8 @@ class Seq10 : public EcIoSequence { Seq10(std::pair obj_size_range, int seed, std::optional> km, std::optional> - mappinglayers); + mappinglayers, + bool check_consistency); Sequence get_id() const override; std::string get_name() const override; diff --git a/src/common/io_exerciser/IoOp.cc b/src/common/io_exerciser/IoOp.cc index 1046d6aaa914a..4b9fc57ce3987 100644 --- a/src/common/io_exerciser/IoOp.cc +++ b/src/common/io_exerciser/IoOp.cc @@ -10,6 +10,7 @@ using DoneOp = ceph::io_exerciser::DoneOp; using BarrierOp = ceph::io_exerciser::BarrierOp; using CreateOp = ceph::io_exerciser::CreateOp; using RemoveOp = ceph::io_exerciser::RemoveOp; +using ConsistencyOp = ceph::io_exerciser::ConsistencyOp; using SingleReadOp = ceph::io_exerciser::SingleReadOp; using DoubleReadOp = ceph::io_exerciser::DoubleReadOp; using TripleReadOp = ceph::io_exerciser::TripleReadOp; @@ -98,6 +99,16 @@ ceph::io_exerciser::ReadWriteOp::ReadWriteOp( } } +ConsistencyOp::ConsistencyOp() : TestOp() {} + +std::unique_ptr ConsistencyOp::generate() { + return std::make_unique(); +} + +std::string ConsistencyOp::to_string(uint64_t block_size) const { + return "Consistency"; +} + template std::string ceph::io_exerciser::ReadWriteOp::to_string( uint64_t block_size) const { diff --git a/src/common/io_exerciser/IoOp.h b/src/common/io_exerciser/IoOp.h index 0cfc6c0230926..ae58f6c40cd50 100644 --- a/src/common/io_exerciser/IoOp.h +++ b/src/common/io_exerciser/IoOp.h @@ -61,6 +61,13 @@ class RemoveOp : public TestOp { std::string to_string(uint64_t block_size) const override; }; +class ConsistencyOp : public TestOp { + public: + ConsistencyOp(); + static std::unique_ptr generate(); + std::string to_string(uint64_t block_size) const override; + }; + template class ReadWriteOp : public TestOp { public: diff --git a/src/common/io_exerciser/IoSequence.cc b/src/common/io_exerciser/IoSequence.cc index 97289238fe6b7..2d4b80aa32c9b 100644 --- a/src/common/io_exerciser/IoSequence.cc +++ b/src/common/io_exerciser/IoSequence.cc @@ -2,6 +2,7 @@ #include using IoOp = ceph::io_exerciser::IoOp; +using OpType = ceph::io_exerciser::OpType; using Sequence = ceph::io_exerciser::Sequence; using IoSequence = ceph::io_exerciser::IoSequence; @@ -65,54 +66,58 @@ bool IoSequence::is_supported(Sequence sequence) const { } std::unique_ptr IoSequence::generate_sequence( - Sequence s, std::pair obj_size_range, int seed) { + Sequence s, std::pair obj_size_range, int seed, bool check_consistency) { switch (s) { case Sequence::SEQUENCE_SEQ0: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ1: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ2: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ3: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ4: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ5: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ6: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ7: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ8: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ9: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ10: ceph_abort_msg( "Sequence 10 only supported for erasure coded pools " "through the EcIoSequence interface"); return nullptr; case Sequence::SEQUENCE_SEQ11: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ12: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ13: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); case Sequence::SEQUENCE_SEQ14: - return std::make_unique(obj_size_range, seed); + return std::make_unique(obj_size_range, seed, check_consistency); default: break; } return nullptr; } -IoSequence::IoSequence(std::pair obj_size_range, int seed) +IoSequence::IoSequence(std::pair obj_size_range, int seed, bool check_consistency) : min_obj_size(obj_size_range.first), max_obj_size(obj_size_range.second), create(true), barrier(false), done(false), remove(false), + consistency(false), + consistency_in_progress(false), + consistency_request_sent(false), + check_consistency(check_consistency), obj_size(min_obj_size), step(-1), seed(seed) { @@ -161,6 +166,26 @@ std::unique_ptr IoSequence::increment_object_size() { return BarrierOp::generate(); } +std::unique_ptr IoSequence::process_remove() { + if (check_consistency) { + if (!consistency_in_progress) { + consistency_in_progress = true; + consistency_request_sent = true; + return ConsistencyOp::generate(); + } + + if (consistency_request_sent) { + consistency_request_sent = false; + return BarrierOp::generate(); + } + } + + // Getting here means consistency and barriers have been applied if required + remove = false; + consistency_in_progress = false; + return RemoveOp::generate(); +} + Sequence IoSequence::getNextSupportedSequenceId() const { Sequence sequence = get_id(); ++sequence; @@ -172,17 +197,20 @@ Sequence IoSequence::getNextSupportedSequenceId() const { return Sequence::SEQUENCE_END; } - std::unique_ptr IoSequence::next() { step++; if (remove) { - remove = false; - return RemoveOp::generate(); + return process_remove(); } if (barrier) { barrier = false; return BarrierOp::generate(); } + if (consistency) { + consistency = false; + barrier = true; + return ConsistencyOp::generate(); + } if (done) { return DoneOp::generate(); } @@ -194,8 +222,8 @@ std::unique_ptr IoSequence::next() { return _next(); } -ceph::io_exerciser::Seq0::Seq0(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), offset(0) { +ceph::io_exerciser::Seq0::Seq0(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset(0) { select_random_object_size(); length = 1 + rng(obj_size - 1); } @@ -226,8 +254,8 @@ std::unique_ptr ceph::io_exerciser::Seq0::_next() { return r; } -ceph::io_exerciser::Seq1::Seq1(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed) { +ceph::io_exerciser::Seq1::Seq1(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency) { select_random_object_size(); count = 3 * obj_size; } @@ -258,8 +286,8 @@ std::unique_ptr ceph::io_exerciser::Seq1::_next() { } } -ceph::io_exerciser::Seq2::Seq2(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), offset(0), length(0) {} +ceph::io_exerciser::Seq2::Seq2(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset(0), length(0) {} Sequence ceph::io_exerciser::Seq2::get_id() const { return Sequence::SEQUENCE_SEQ2; @@ -283,8 +311,8 @@ std::unique_ptr ceph::io_exerciser::Seq2::_next() { return SingleReadOp::generate(offset, length); } -ceph::io_exerciser::Seq3::Seq3(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), offset1(0), offset2(0) { +ceph::io_exerciser::Seq3::Seq3(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset1(0), offset2(0) { set_min_object_size(2); } @@ -310,8 +338,8 @@ std::unique_ptr ceph::io_exerciser::Seq3::_next() { return DoubleReadOp::generate(offset1, 1, offset1 + offset2, 1); } -ceph::io_exerciser::Seq4::Seq4(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), offset1(0), offset2(1) { +ceph::io_exerciser::Seq4::Seq4(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset1(0), offset2(1) { set_min_object_size(3); } @@ -338,8 +366,8 @@ std::unique_ptr ceph::io_exerciser::Seq4::_next() { (offset1 * 2 + offset2) / 2, 1); } -ceph::io_exerciser::Seq5::Seq5(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), +ceph::io_exerciser::Seq5::Seq5(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset(0), length(1), doneread(false), @@ -358,6 +386,7 @@ std::unique_ptr ceph::io_exerciser::Seq5::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; + consistency = check_consistency; return BarrierOp::generate(); } doneread = true; @@ -379,8 +408,8 @@ std::unique_ptr ceph::io_exerciser::Seq5::_next() { return r; } -ceph::io_exerciser::Seq6::Seq6(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), +ceph::io_exerciser::Seq6::Seq6(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset(0), length(1), doneread(false), @@ -399,6 +428,7 @@ std::unique_ptr ceph::io_exerciser::Seq6::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; + consistency = check_consistency; return BarrierOp::generate(); } doneread = true; @@ -423,8 +453,8 @@ std::unique_ptr ceph::io_exerciser::Seq6::_next() { return r; } -ceph::io_exerciser::Seq7::Seq7(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed) { +ceph::io_exerciser::Seq7::Seq7(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency) { set_min_object_size(2); offset = obj_size; } @@ -441,6 +471,7 @@ std::unique_ptr ceph::io_exerciser::Seq7::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; + consistency = check_consistency; return BarrierOp::generate(); } doneread = true; @@ -462,8 +493,8 @@ std::unique_ptr ceph::io_exerciser::Seq7::_next() { return DoubleWriteOp::generate(offset, 1, obj_size / 2, 1); } -ceph::io_exerciser::Seq8::Seq8(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), offset1(0), offset2(1) { +ceph::io_exerciser::Seq8::Seq8(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset1(0), offset2(1) { set_min_object_size(3); } @@ -479,6 +510,7 @@ std::unique_ptr ceph::io_exerciser::Seq8::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; + consistency = check_consistency; return BarrierOp::generate(); } doneread = true; @@ -501,8 +533,8 @@ std::unique_ptr ceph::io_exerciser::Seq8::_next() { (offset1 * 2 + offset2) / 2, 1); } -ceph::io_exerciser::Seq9::Seq9(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), offset(0), length(0) {} +ceph::io_exerciser::Seq9::Seq9(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), offset(0), length(0) {} Sequence ceph::io_exerciser::Seq9::get_id() const { return Sequence::SEQUENCE_SEQ9; @@ -516,6 +548,7 @@ std::unique_ptr ceph::io_exerciser::Seq9::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; + consistency = check_consistency; return BarrierOp::generate(); } doneread = true; @@ -537,8 +570,8 @@ std::unique_ptr ceph::io_exerciser::Seq9::_next() { return SingleWriteOp::generate(offset, length); } -ceph::io_exerciser::Seq11::Seq11(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), +ceph::io_exerciser::Seq11::Seq11(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), count(0), doneread(false), donebarrier(false) {} @@ -570,8 +603,8 @@ std::unique_ptr ceph::io_exerciser::Seq11::_next() { return SingleAppendOp::generate(obj_size); } -ceph::io_exerciser::Seq12::Seq12(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), count(0), overlap(1), doneread(false) {} +ceph::io_exerciser::Seq12::Seq12(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), count(0), overlap(1), doneread(false) {} Sequence ceph::io_exerciser::Seq12::get_id() const { return Sequence::SEQUENCE_SEQ12; @@ -606,8 +639,8 @@ std::unique_ptr ceph::io_exerciser::Seq12::_next() { obj_size + overlap); } -ceph::io_exerciser::Seq13::Seq13(std::pair obj_size_range, int seed) - : IoSequence(obj_size_range, seed), count(0), gap(1), doneread(false) { +ceph::io_exerciser::Seq13::Seq13(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(obj_size_range, seed, check_consistency), count(0), gap(1), doneread(false) { set_min_object_size(2); } @@ -643,8 +676,8 @@ std::unique_ptr ceph::io_exerciser::Seq13::_next() { return SingleWriteOp::generate((count * obj_size) + gap, obj_size - gap); } -ceph::io_exerciser::Seq14::Seq14(std::pair obj_size_range, int seed) - : IoSequence(std::make_pair(0, obj_size_range.second), seed), +ceph::io_exerciser::Seq14::Seq14(std::pair obj_size_range, int seed, bool check_consistency) + : IoSequence(std::make_pair(0, obj_size_range.second), seed, check_consistency), offset(0), step(1) { startrng = std::default_random_engine(seed); @@ -676,6 +709,8 @@ std::string ceph::io_exerciser::Seq14::get_name() const { std::unique_ptr ceph::io_exerciser::Seq14::_next() { if (offset >= target_obj_size) { if (!doneread) { + consistency = check_consistency; + barrier = check_consistency; doneread = true; return SingleReadOp::generate(0, current_size); } diff --git a/src/common/io_exerciser/IoSequence.h b/src/common/io_exerciser/IoSequence.h index c0680408db134..86282cabcc1f7 100644 --- a/src/common/io_exerciser/IoSequence.h +++ b/src/common/io_exerciser/IoSequence.h @@ -74,7 +74,7 @@ class IoSequence { virtual bool is_supported(Sequence sequence) const; static std::unique_ptr generate_sequence( - Sequence s, std::pair obj_size_range, int seed); + Sequence s, std::pair obj_size_range, int seed, bool check_consistency); protected: uint64_t min_obj_size; @@ -83,13 +83,17 @@ class IoSequence { bool barrier; bool done; bool remove; + bool consistency; + bool consistency_in_progress; + bool consistency_request_sent; + bool check_consistency; uint64_t obj_size; int step; int seed; ceph::util::random_number_generator rng = ceph::util::random_number_generator(); - IoSequence(std::pair obj_size_range, int seed); + IoSequence(std::pair obj_size_range, int seed, bool check_consistency); virtual std::unique_ptr _next() = 0; @@ -97,11 +101,12 @@ class IoSequence { void set_max_object_size(uint64_t size); void select_random_object_size(); std::unique_ptr increment_object_size(); + std::unique_ptr process_remove(); }; class Seq0 : public IoSequence { public: - Seq0(std::pair obj_size_range, int seed); + Seq0(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -114,7 +119,7 @@ class Seq0 : public IoSequence { class Seq1 : public IoSequence { public: - Seq1(std::pair obj_size_range, int seed); + Seq1(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -126,7 +131,7 @@ class Seq1 : public IoSequence { class Seq2 : public IoSequence { public: - Seq2(std::pair obj_size_range, int seed); + Seq2(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -139,7 +144,7 @@ class Seq2 : public IoSequence { class Seq3 : public IoSequence { public: - Seq3(std::pair obj_size_range, int seed); + Seq3(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -152,7 +157,7 @@ class Seq3 : public IoSequence { class Seq4 : public IoSequence { public: - Seq4(std::pair obj_size_range, int seed); + Seq4(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -165,7 +170,7 @@ class Seq4 : public IoSequence { class Seq5 : public IoSequence { public: - Seq5(std::pair obj_size_range, int seed); + Seq5(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -180,7 +185,7 @@ class Seq5 : public IoSequence { class Seq6 : public IoSequence { public: - Seq6(std::pair obj_size_range, int seed); + Seq6(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -195,7 +200,7 @@ class Seq6 : public IoSequence { class Seq7 : public IoSequence { public: - Seq7(std::pair obj_size_range, int seed); + Seq7(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -209,7 +214,7 @@ class Seq7 : public IoSequence { class Seq8 : public IoSequence { public: - Seq8(std::pair obj_size_range, int seed); + Seq8(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -230,7 +235,7 @@ class Seq9 : public IoSequence { bool donebarrier = false; public: - Seq9(std::pair obj_size_range, int seed); + Seq9(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -244,7 +249,7 @@ class Seq11 : public IoSequence { bool donebarrier = false; public: - Seq11(std::pair obj_size_range, int seed); + Seq11(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -259,7 +264,7 @@ class Seq12 : public IoSequence { bool donebarrier = false; public: - Seq12(std::pair obj_size_range, int seed); + Seq12(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -274,7 +279,7 @@ class Seq13 : public IoSequence { bool donebarrier = false; public: - Seq13(std::pair obj_size_range, int seed); + Seq13(std::pair obj_size_range, int seed, bool check_consistency); Sequence get_id() const override; std::string get_name() const override; @@ -293,7 +298,7 @@ class Seq14 : public IoSequence { bool doneread = false; public: - Seq14(std::pair obj_size_range, int seed); + Seq14(std::pair obj_size_range, int seed, bool check_consistency); void setup_starts(); Sequence get_id() const override; diff --git a/src/common/io_exerciser/OpType.h b/src/common/io_exerciser/OpType.h index 6a7e4604407e5..3c7f535839c31 100644 --- a/src/common/io_exerciser/OpType.h +++ b/src/common/io_exerciser/OpType.h @@ -17,6 +17,7 @@ enum class OpType { Barrier, // Barrier - all prior I/Os must complete Create, // Create object and pattern with data Remove, // Remove object + Consistency, // Check consistency of an object Read, // Read Read2, // Two reads in a single op Read3, // Three reads in a single op @@ -59,6 +60,8 @@ struct fmt::formatter { return fmt::format_to(ctx.out(), "Create"); case ceph::io_exerciser::OpType::Remove: return fmt::format_to(ctx.out(), "Remove"); + case ceph::io_exerciser::OpType::Consistency: + return fmt::format_to(ctx.out(), "Consistency"); case ceph::io_exerciser::OpType::Read: return fmt::format_to(ctx.out(), "Read"); case ceph::io_exerciser::OpType::Read2: diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc index e411ddede4907..1b07d4d12ae15 100644 --- a/src/common/io_exerciser/RadosIo.cc +++ b/src/common/io_exerciser/RadosIo.cc @@ -10,6 +10,7 @@ #include "common/json/OSDStructures.h" using RadosIo = ceph::io_exerciser::RadosIo; +using ConsistencyChecker = ceph::consistency::ConsistencyChecker; namespace { template @@ -48,6 +49,7 @@ RadosIo::RadosIo(librados::Rados& rados, boost::asio::io_context& asio, om(std::make_unique(oid, block_size, seed)), db(data_generation::DataGenerator::create_generator( data_generation::GenerationType::HeaderedSeededRandom, *om)), + cc(std::make_unique(rados, asio, pool)), pool(pool), cached_shard_order(cached_shard_order), threads(threads), @@ -188,6 +190,16 @@ void RadosIo::applyIoOp(IoOp& op) { std::move(wop), 0, nullptr, remove_cb); break; } + + case OpType::Consistency: { + start_io(); + bool is_consistent = + cc->single_read_and_check_consistency(oid, block_size, 0, 0); + ceph_assert(is_consistent); + finish_io(); + break; + } + case OpType::Read: [[fallthrough]]; case OpType::Read2: @@ -496,4 +508,4 @@ void RadosIo::applyInjectOp(IoOp& op) { fmt::format("Unsupported inject operation ({})", op.getOpType())); break; } -} +} \ No newline at end of file diff --git a/src/common/io_exerciser/RadosIo.h b/src/common/io_exerciser/RadosIo.h index 1837b75725ae0..c9add85bb82dc 100644 --- a/src/common/io_exerciser/RadosIo.h +++ b/src/common/io_exerciser/RadosIo.h @@ -1,6 +1,7 @@ #pragma once #include "ObjectModel.h" +#include "erasure-code/consistency/ConsistencyChecker.h" /* Overview * @@ -25,6 +26,7 @@ class RadosIo : public Model { boost::asio::io_context& asio; std::unique_ptr om; std::unique_ptr db; + std::unique_ptr cc; std::string pool; std::optional> cached_shard_order; int threads; diff --git a/src/test/osd/CMakeLists.txt b/src/test/osd/CMakeLists.txt index a777ee9179aa3..cbe5113d3829f 100644 --- a/src/test/osd/CMakeLists.txt +++ b/src/test/osd/CMakeLists.txt @@ -23,7 +23,7 @@ add_executable(ceph_test_rados_io_sequence ${CMAKE_CURRENT_SOURCE_DIR}/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc) add_dependencies(ceph_test_rados_io_sequence erasure_code_plugins) target_link_libraries(ceph_test_rados_io_sequence - librados global object_io_exerciser json_structures) + librados global object_io_exerciser json_structures ec_consistency) install(TARGETS ceph_test_rados_io_sequence DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc index 396ec815afaff..a99f3de674c45 100644 --- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc +++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc @@ -195,6 +195,8 @@ po::options_description get_options_description() { "number of objects to exercise in parallel")( "testrecovery", "Inject errors during sequences to test recovery processes of OSDs")( + "checkconsistency", + "Test objects for consistency during IO sequences. Disabled by default.")( "interactive", "interactive mode, execute IO commands from stdin")( "allow_pool_autoscaling", "Allows pool autoscaling. Disabled by default.")( @@ -947,8 +949,9 @@ ceph::io_sequence::tester::TestObject::TestObject( SelectObjectSize& sos, SelectNumThreads& snt, SelectSeqRange& ssr, ceph::util::random_number_generator& rng, ceph::mutex& lock, ceph::condition_variable& cond, bool dryrun, bool verbose, - std::optional seqseed, bool testrecovery) - : rng(rng), verbose(verbose), seqseed(seqseed), testrecovery(testrecovery) { + std::optional seqseed, bool testrecovery, bool checkconsistency) + : rng(rng), verbose(verbose), seqseed(seqseed), + testrecovery(testrecovery), checkconsistency(checkconsistency) { if (dryrun) { exerciser_model = std::make_unique( oid, sbs.select(), rng()); @@ -1001,10 +1004,10 @@ ceph::io_sequence::tester::TestObject::TestObject( if (testrecovery) { seq = ceph::io_exerciser::EcIoSequence::generate_sequence( curseq, obj_size_range, pool_km, pool_mappinglayers, - seqseed.value_or(rng())); + seqseed.value_or(rng()), checkconsistency); } else { seq = ceph::io_exerciser::IoSequence::generate_sequence( - curseq, obj_size_range, seqseed.value_or(rng())); + curseq, obj_size_range, seqseed.value_or(rng()), checkconsistency); } op = seq->next(); @@ -1040,10 +1043,10 @@ bool ceph::io_sequence::tester::TestObject::next() { if (testrecovery) { seq = ceph::io_exerciser::EcIoSequence::generate_sequence( curseq, obj_size_range, pool_km, pool_mappinglayers, - seqseed.value_or(rng())); + seqseed.value_or(rng()), checkconsistency); } else { seq = ceph::io_exerciser::IoSequence::generate_sequence( - curseq, obj_size_range, seqseed.value_or(rng())); + curseq, obj_size_range, seqseed.value_or(rng()), checkconsistency); } dout(0) << "== " << exerciser_model->get_oid() << " " << curseq << " " @@ -1098,6 +1101,7 @@ ceph::io_sequence::tester::TestRunner::TestRunner( object_name = vm["object"].as(); interactive = vm.contains("interactive"); testrecovery = vm.contains("testrecovery"); + checkconsistency = vm.contains("checkconsistency"); allow_pool_autoscaling = vm.contains("allow_pool_autoscaling"); allow_pool_balancer = vm.contains("allow_pool_balancer"); @@ -1131,7 +1135,7 @@ void ceph::io_sequence::tester::TestRunner::help() { } void ceph::io_sequence::tester::TestRunner::list_sequence(bool testrecovery) { - // List seqeunces + // List sequences std::pair obj_size_range = sos.select(); ceph::io_exerciser::Sequence s = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN; std::unique_ptr seq; @@ -1151,11 +1155,11 @@ void ceph::io_sequence::tester::TestRunner::list_sequence(bool testrecovery) { do { if (testrecovery) { seq = ceph::io_exerciser::EcIoSequence::generate_sequence( - s, obj_size_range, km, mappinglayers, seqseed.value_or(rng())); + s, obj_size_range, km, mappinglayers, seqseed.value_or(rng()), checkconsistency); } else { seq = ceph::io_exerciser::IoSequence::generate_sequence( - s, obj_size_range, seqseed.value_or(rng())); + s, obj_size_range, seqseed.value_or(rng()), checkconsistency); } dout(0) << s << " " << seq->get_name_with_seqseed() << dendl; @@ -1413,7 +1417,7 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() { test_objects.push_back( std::make_shared( name, rados, asio, sbs, spo, sos, snt, ssr, rng, lock, cond, - dryrun, verbose, seqseed, testrecovery)); + dryrun, verbose, seqseed, testrecovery, checkconsistency)); } catch (const std::runtime_error &e) { std::cerr << "Error: " << e.what() << std::endl; diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h index 16fe6d7b778d1..26f2cd5d9e0a1 100644 --- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h +++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h @@ -443,7 +443,8 @@ class TestObject { bool dryrun, bool verbose, std::optional seqseed, - bool testRecovery); + bool testRecovery, + bool checkConsistency); int get_num_io(); bool readyForIo(); @@ -466,6 +467,7 @@ class TestObject { std::optional> pool_mappinglayers; bool testrecovery; + bool checkconsistency; }; class TestRunner { @@ -504,6 +506,7 @@ class TestRunner { bool interactive; bool testrecovery; + bool checkconsistency; bool allow_pool_autoscaling; bool allow_pool_balancer; -- 2.39.5