From e2ef12525f52eda475b1434110388f629a3130c9 Mon Sep 17 00:00:00 2001 From: JonBailey1993 Date: Tue, 15 Oct 2024 16:00:33 +0100 Subject: [PATCH] common/io_exerciser: Add simple sequences for testing error injects Add sequences to test IOs with simple error injects, along with some small fixes for previous error inject implementation. Signed-off-by: Jon Bailey --- src/common/io_exerciser/CMakeLists.txt | 1 + src/common/io_exerciser/DataGenerator.h | 2 +- src/common/io_exerciser/EcIoSequence.cc | 296 ++++++++++++++++++++ src/common/io_exerciser/EcIoSequence.h | 71 +++++ src/common/io_exerciser/IoSequence.cc | 105 ++++++- src/common/io_exerciser/IoSequence.h | 20 +- src/common/io_exerciser/JsonStructures.cc | 171 ++++++++++- src/common/io_exerciser/JsonStructures.h | 71 ++++- src/common/io_exerciser/OpType.h | 8 + src/common/io_exerciser/RadosIo.cc | 67 ++++- src/test/osd/ceph_test_rados_io_sequence.cc | 123 ++++++-- src/test/osd/ceph_test_rados_io_sequence.h | 16 +- 12 files changed, 888 insertions(+), 63 deletions(-) create mode 100644 src/common/io_exerciser/EcIoSequence.cc create mode 100644 src/common/io_exerciser/EcIoSequence.h diff --git a/src/common/io_exerciser/CMakeLists.txt b/src/common/io_exerciser/CMakeLists.txt index 930cfec9ab5..ccad61b57a2 100644 --- a/src/common/io_exerciser/CMakeLists.txt +++ b/src/common/io_exerciser/CMakeLists.txt @@ -6,6 +6,7 @@ add_library(object_io_exerciser STATIC ObjectModel.cc RadosIo.cc JsonStructures.cc + EcIoSequence.cc ) target_link_libraries(object_io_exerciser diff --git a/src/common/io_exerciser/DataGenerator.h b/src/common/io_exerciser/DataGenerator.h index 1e5784a54cc..886454fdfa4 100644 --- a/src/common/io_exerciser/DataGenerator.h +++ b/src/common/io_exerciser/DataGenerator.h @@ -65,7 +65,7 @@ namespace ceph { : DataGenerator(model) {} virtual bufferptr generate_block(uint64_t offset); - virtual bufferlist generate_data(uint64_t length, uint64_t offset); + virtual bufferlist generate_data(uint64_t length, uint64_t offset) override; virtual bufferptr generate_wrong_block(uint64_t offset); virtual bufferlist generate_wrong_data(uint64_t offset, uint64_t length) override; }; diff --git a/src/common/io_exerciser/EcIoSequence.cc b/src/common/io_exerciser/EcIoSequence.cc new file mode 100644 index 00000000000..cc605de0e00 --- /dev/null +++ b/src/common/io_exerciser/EcIoSequence.cc @@ -0,0 +1,296 @@ +#include "EcIoSequence.h" + +#include + +using IoOp = ceph::io_exerciser::IoOp; +using Sequence = ceph::io_exerciser::Sequence; +using IoSequence = ceph::io_exerciser::IoSequence; +using EcIoSequence = ceph::io_exerciser::EcIoSequence; +using ReadInjectSequence = ceph::io_exerciser::ReadInjectSequence; + +bool EcIoSequence::is_supported(Sequence sequence) const +{ + return true; +} + +std::unique_ptr EcIoSequence::generate_sequence(Sequence sequence, + std::pair obj_size_range, + int k, + int m, + int seed) +{ + switch(sequence) + { + case Sequence::SEQUENCE_SEQ0: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ1: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ2: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ3: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ4: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ5: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ6: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ7: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ8: + [[ fallthrough ]]; + case Sequence::SEQUENCE_SEQ9: + return std::make_unique(obj_size_range, seed, + sequence, k, m); + case Sequence::SEQUENCE_SEQ10: + return std::make_unique(obj_size_range, seed, k, m); + default: + ceph_abort_msg("Unrecognised sequence"); + } +} + +EcIoSequence::EcIoSequence(std::pair obj_size_range, int seed) : + IoSequence(obj_size_range, seed), + setup_inject(false), clear_inject(false), shard_to_inject(std::nullopt) +{ + +} + +void EcIoSequence::select_random_data_shard_to_inject_read_error(int k, int m) +{ + shard_to_inject = rng(k - 1); + setup_inject = true; +} + +void EcIoSequence::select_random_data_shard_to_inject_write_error(int k, int m) +{ + // Write errors do not support injecting to the primary OSD + shard_to_inject = rng(1, k - 1); + setup_inject = true; +} + +void EcIoSequence::select_random_shard_to_inject_read_error(int k, int m) +{ + shard_to_inject = rng(k + m - 1); + setup_inject = true; +} + +void EcIoSequence::select_random_shard_to_inject_write_error(int k, int m) +{ + // Write errors do not support injecting to the primary OSD + shard_to_inject = rng(1, k + m - 1); + setup_inject = true; +} + +void EcIoSequence::generate_random_read_inject_type() +{ + inject_op_type = static_cast(rng(static_cast(InjectOpType::ReadEIO), + static_cast(InjectOpType::ReadMissingShard))); +} + +void EcIoSequence::generate_random_write_inject_type() +{ + inject_op_type = static_cast(rng(static_cast(InjectOpType::WriteFailAndRollback), + static_cast(InjectOpType::WriteOSDAbort))); +} + +ceph::io_exerciser::ReadInjectSequence::ReadInjectSequence(std::pair obj_size_range, + int seed, + Sequence s, + int k, int m) : + EcIoSequence(obj_size_range, seed) +{ + child_sequence = IoSequence::generate_sequence(s, obj_size_range, seed); + select_random_data_shard_to_inject_read_error(k, m); + generate_random_read_inject_type(); +} + +Sequence ceph::io_exerciser::ReadInjectSequence::get_id() const +{ + return child_sequence->get_id(); +} + +std::string ceph::io_exerciser::ReadInjectSequence::get_name() const +{ + return child_sequence->get_name() + + " running with read errors injected on shard " + + std::to_string(*shard_to_inject); +} + +std::unique_ptr ReadInjectSequence::next() +{ + step++; + + if (nextOp) + { + std::unique_ptr retOp = nullptr; + nextOp.swap(retOp); + return retOp; + } + + std::unique_ptr childOp = child_sequence->next(); + + switch(childOp->getOpType()) + { + case OpType::Remove: + nextOp.swap(childOp); + switch(inject_op_type) + { + case InjectOpType::ReadEIO: + return ClearReadErrorInjectOp::generate(*shard_to_inject, 0); + case InjectOpType::ReadMissingShard: + return ClearReadErrorInjectOp::generate(*shard_to_inject, 1); + case InjectOpType::WriteFailAndRollback: + return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0); + case InjectOpType::WriteOSDAbort: + return ClearWriteErrorInjectOp::generate(*shard_to_inject, 3); + case InjectOpType::None: + [[ fallthrough ]]; + default: + ceph_abort_msg("Unsupported operation"); + } + break; + case OpType::Create: + switch(inject_op_type) + { + case InjectOpType::ReadEIO: + nextOp = InjectReadErrorOp::generate(*shard_to_inject, 0, 0, std::numeric_limits::max()); + break; + case InjectOpType::ReadMissingShard: + nextOp = InjectReadErrorOp::generate(*shard_to_inject, 1, 0, std::numeric_limits::max()); + break; + case InjectOpType::WriteFailAndRollback: + nextOp = InjectWriteErrorOp::generate(*shard_to_inject, 0, 0, std::numeric_limits::max()); + break; + case InjectOpType::WriteOSDAbort: + nextOp = InjectWriteErrorOp::generate(*shard_to_inject, 3, 0, std::numeric_limits::max()); + break; + case InjectOpType::None: + [[ fallthrough ]]; + default: + ceph_abort_msg("Unsupported operation"); + } + break; + default: + // Do nothing in default case + break; + } + + return childOp; +} + +std::unique_ptr ceph::io_exerciser::ReadInjectSequence::_next() +{ + ceph_abort_msg("Should not reach this point, " + "this sequence should only consume complete sequences"); + + return DoneOp::generate(); +} + + + +ceph::io_exerciser::Seq10::Seq10(std::pair obj_size_range, int seed, + int k, int m) : + EcIoSequence(obj_size_range, seed), offset(0), length(1), + failed_write_done(false), read_done(false), successful_write_done(false), + test_all_lengths(false), // Only test length(1) due to time constraints + test_all_sizes(false) // Only test obj_size(rand()) due to time constraints +{ + select_random_shard_to_inject_write_error(k, m); + // We will inject specifically as part of our sequence in this sequence + setup_inject = false; + if (!test_all_sizes) + { + select_random_object_size(); + } +} + +Sequence ceph::io_exerciser::Seq10::get_id() const +{ + return Sequence::SEQUENCE_SEQ10; +} + +std::string ceph::io_exerciser::Seq10::get_name() const +{ + return "Sequential writes of length " + std::to_string(length) + + " with queue depth 1" + " first injecting a failed write and read it to ensure it rolls back, then" + " successfully writing the data and reading the write the ensure it is applied"; +} + +std::unique_ptr ceph::io_exerciser::Seq10::_next() +{ + if (!inject_error_done) + { + inject_error_done = true; + return InjectWriteErrorOp::generate(*shard_to_inject, 0, 0, + std::numeric_limits::max()); + } + else if (!failed_write_done) + { + failed_write_done = true; + read_done = false; + barrier = true; + return SingleFailedWriteOp::generate(offset, length); + } + else if (failed_write_done && !read_done) + { + read_done = true; + barrier = true; + return SingleReadOp::generate(offset, length); + } + else if (!clear_inject_done) + { + clear_inject_done = true; + return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0); + } + else if (!successful_write_done) + { + successful_write_done = true; + read_done = false; + barrier = true; + return SingleWriteOp::generate(offset, length); + } + else if (successful_write_done && !read_done) + { + read_done = true; + return SingleReadOp::generate(offset, length); + } + else if (successful_write_done && read_done) + { + offset++; + inject_error_done = false; + failed_write_done = false; + read_done = false; + clear_inject_done = false; + successful_write_done = false; + + if (offset + length >= obj_size) { + if (!test_all_lengths) + { + done = true; + return BarrierOp::generate(); + } + + offset = 0; + length++; + if (length > obj_size) { + if (!test_all_sizes) + { + done = true; + return BarrierOp::generate(); + } + + length = 1; + return increment_object_size(); + } + } + + return BarrierOp::generate(); + } + else + { + ceph_abort_msg("Sequence in undefined state. Aborting"); + return DoneOp::generate(); + } +} \ No newline at end of file diff --git a/src/common/io_exerciser/EcIoSequence.h b/src/common/io_exerciser/EcIoSequence.h new file mode 100644 index 00000000000..acfbb34ac27 --- /dev/null +++ b/src/common/io_exerciser/EcIoSequence.h @@ -0,0 +1,71 @@ +#include "IoSequence.h" + +namespace ceph +{ + namespace io_exerciser + { + class EcIoSequence : public IoSequence + { + public: + virtual bool is_supported(Sequence sequence) const override; + static std::unique_ptr + generate_sequence(Sequence s, + std::pair obj_size_range, + int k, + int m, + int seed ); + + protected: + bool setup_inject; + bool clear_inject; + std::optional shard_to_inject; + InjectOpType inject_op_type; + + EcIoSequence(std::pair obj_size_range, int seed); + + // Writes cannot be sent to injected on shard zero, so selections seperated out + void select_random_data_shard_to_inject_read_error(int k, int m); + void select_random_data_shard_to_inject_write_error(int k, int m); + void select_random_shard_to_inject_read_error(int k, int m); + void select_random_shard_to_inject_write_error(int k, int m); + void generate_random_read_inject_type(); + void generate_random_write_inject_type(); + }; + + class ReadInjectSequence : public EcIoSequence + { + public: + ReadInjectSequence(std::pair obj_size_range, int seed, Sequence s, int k, int m); + + Sequence get_id() const override; + std::string get_name() const override; + virtual std::unique_ptr next() override; + std::unique_ptr _next() override; + + private: + std::unique_ptr child_sequence; + std::unique_ptr nextOp; + }; + + class Seq10: public EcIoSequence { + public: + Seq10(std::pair obj_size_range, int seed, int k, int m); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset; + uint64_t length; + + bool inject_error_done; + bool failed_write_done; + bool read_done; + bool clear_inject_done; + bool successful_write_done; + bool test_all_lengths; + bool test_all_sizes; + }; + } +} \ No newline at end of file diff --git a/src/common/io_exerciser/IoSequence.cc b/src/common/io_exerciser/IoSequence.cc index d4b7af40b41..f14f0d117da 100644 --- a/src/common/io_exerciser/IoSequence.cc +++ b/src/common/io_exerciser/IoSequence.cc @@ -38,6 +38,9 @@ std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& s case Sequence::SEQUENCE_SEQ9: os << "SEQUENCE_SEQ9"; break; + case Sequence::SEQUENCE_SEQ10: + os << "SEQUENCE_SEQ10"; + break; case Sequence::SEQUENCE_END: os << "SEQUENCE_END"; break; @@ -45,6 +48,11 @@ std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& s return os; } +bool IoSequence::is_supported(Sequence sequence) const +{ + return sequence != Sequence::SEQUENCE_SEQ10; +} + std::unique_ptr IoSequence::generate_sequence(Sequence s, std::pair obj_size_range, int seed) @@ -70,6 +78,10 @@ std::unique_ptr IoSequence::generate_sequence(Sequence s, return std::make_unique(obj_size_range, seed); case Sequence::SEQUENCE_SEQ9: return std::make_unique(obj_size_range, seed); + case Sequence::SEQUENCE_SEQ10: + ceph_abort_msg("Sequence 10 only supported for erasure coded pools " + "through the EcIoSequence interface"); + return nullptr; default: break; } @@ -77,7 +89,7 @@ std::unique_ptr IoSequence::generate_sequence(Sequence s, } IoSequence::IoSequence(std::pair obj_size_range, - int seed) : + int seed) : min_obj_size(obj_size_range.first), max_obj_size(obj_size_range.second), create(true), barrier(false), done(false), remove(false), obj_size(min_obj_size), step(-1), seed(seed) @@ -85,6 +97,11 @@ IoSequence::IoSequence(std::pair obj_size_range, rng.seed(seed); } +std::string ceph::io_exerciser::IoSequence::get_name_with_seqseed() const +{ + return get_name() + " (seqseed " + std::to_string(get_seed()) + ")"; +} + int IoSequence::get_step() const { return step; @@ -133,6 +150,22 @@ std::unique_ptr IoSequence::increment_object_size() return BarrierOp::generate(); } +Sequence IoSequence::getNextSupportedSequenceId() const +{ + Sequence sequence = get_id(); + ++sequence; + for (;sequence < Sequence::SEQUENCE_END; + ++sequence) + { + if (is_supported(sequence)) + { + return sequence; + } + } + + return Sequence::SEQUENCE_END; +} + std::unique_ptr IoSequence::next() { step++; @@ -164,10 +197,15 @@ ceph::io_exerciser::Seq0::Seq0(std::pair obj_size_range, int seed) : length = 1 + rng(obj_size - 1); } +Sequence ceph::io_exerciser::Seq0::get_id() const +{ + return Sequence::SEQUENCE_SEQ0; +} + std::string ceph::io_exerciser::Seq0::get_name() const { return "Sequential reads of length " + std::to_string(length) + - " with queue depth 1 (seqseed " + std::to_string(get_seed()) + ")"; + " with queue depth 1"; } std::unique_ptr ceph::io_exerciser::Seq0::_next() @@ -197,10 +235,14 @@ ceph::io_exerciser::Seq1::Seq1(std::pair obj_size_range, int seed) : count = 3 * obj_size; } +Sequence ceph::io_exerciser::Seq1::get_id() const +{ + return Sequence::SEQUENCE_SEQ1; +} + std::string ceph::io_exerciser::Seq1::get_name() const { - return "Random offset, random length read/write I/O with queue depth 1 (seqseed " - + std::to_string(get_seed()) + ")"; + return "Random offset, random length read/write I/O with queue depth 1"; } std::unique_ptr ceph::io_exerciser::Seq1::_next() @@ -228,7 +270,15 @@ std::unique_ptr ceph::io_exerciser::Seq1::_next() ceph::io_exerciser::Seq2::Seq2(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset(0), length(0) {} + IoSequence(obj_size_range, seed), offset(0), length(0) +{ + +} + +Sequence ceph::io_exerciser::Seq2::get_id() const +{ + return Sequence::SEQUENCE_SEQ2; +} std::string ceph::io_exerciser::Seq2::get_name() const { @@ -258,6 +308,11 @@ ceph::io_exerciser::Seq3::Seq3(std::pair obj_size_range, int seed) : set_min_object_size(2); } +Sequence ceph::io_exerciser::Seq3::get_id() const +{ + return Sequence::SEQUENCE_SEQ3; +} + std::string ceph::io_exerciser::Seq3::get_name() const { return "Permutations of offset 2-region 1-block read I/O"; @@ -286,6 +341,11 @@ ceph::io_exerciser::Seq4::Seq4(std::pair obj_size_range, int seed) : set_min_object_size(3); } +Sequence ceph::io_exerciser::Seq4::get_id() const +{ + return Sequence::SEQUENCE_SEQ4; +} + std::string ceph::io_exerciser::Seq4::get_name() const { return "Permutations of offset 3-region 1-block read I/O"; @@ -310,7 +370,15 @@ std::unique_ptr ceph::io_exerciser::Seq4::_next() ceph::io_exerciser::Seq5::Seq5(std::pair obj_size_range, int seed) : IoSequence(obj_size_range, seed), offset(0), length(1), - doneread(false), donebarrier(false) {} + doneread(false), donebarrier(false) +{ + +} + +Sequence ceph::io_exerciser::Seq5::get_id() const +{ + return Sequence::SEQUENCE_SEQ5; +} std::string ceph::io_exerciser::Seq5::get_name() const { @@ -348,7 +416,15 @@ std::unique_ptr ceph::io_exerciser::Seq5::_next() ceph::io_exerciser::Seq6::Seq6(std::pair obj_size_range, int seed) : IoSequence(obj_size_range, seed), offset(0), length(1), - doneread(false), donebarrier(false) {} + doneread(false), donebarrier(false) +{ + +} + +Sequence ceph::io_exerciser::Seq6::get_id() const +{ + return Sequence::SEQUENCE_SEQ6; +} std::string ceph::io_exerciser::Seq6::get_name() const { @@ -394,6 +470,11 @@ ceph::io_exerciser::Seq7::Seq7(std::pair obj_size_range, int seed) : offset = obj_size; } +Sequence ceph::io_exerciser::Seq7::get_id() const +{ + return Sequence::SEQUENCE_SEQ7; +} + std::string ceph::io_exerciser::Seq7::get_name() const { return "Permutations of offset 2-region 1-block writes"; @@ -433,6 +514,11 @@ ceph::io_exerciser::Seq8::Seq8(std::pair obj_size_range, int seed) : set_min_object_size(3); } +Sequence ceph::io_exerciser::Seq8::get_id() const +{ + return Sequence::SEQUENCE_SEQ8; +} + std::string ceph::io_exerciser::Seq8::get_name() const { return "Permutations of offset 3-region 1-block write I/O"; @@ -472,6 +558,11 @@ ceph::io_exerciser::Seq9::Seq9(std::pair obj_size_range, int seed) : } +Sequence ceph::io_exerciser::Seq9::get_id() const +{ + return Sequence::SEQUENCE_SEQ9; +} + std::string ceph::io_exerciser::Seq9::get_name() const { return "Permutations of offset and length write I/O"; diff --git a/src/common/io_exerciser/IoSequence.h b/src/common/io_exerciser/IoSequence.h index ccce4336c22..46116c5a1c6 100644 --- a/src/common/io_exerciser/IoSequence.h +++ b/src/common/io_exerciser/IoSequence.h @@ -42,7 +42,8 @@ namespace ceph { SEQUENCE_SEQ7, SEQUENCE_SEQ8, SEQUENCE_SEQ9, - // + SEQUENCE_SEQ10, + SEQUENCE_END, SEQUENCE_BEGIN = SEQUENCE_SEQ0 }; @@ -60,12 +61,16 @@ namespace ceph { public: virtual ~IoSequence() = default; + virtual Sequence get_id() const = 0; + virtual std::string get_name_with_seqseed() const; virtual std::string get_name() const = 0; int get_step() const; int get_seed() const; + virtual Sequence getNextSupportedSequenceId() const; virtual std::unique_ptr next(); + virtual bool is_supported(Sequence sequence) const; static std::unique_ptr generate_sequence(Sequence s, std::pair obj_size_range, @@ -98,6 +103,7 @@ namespace ceph { public: Seq0(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; @@ -110,8 +116,9 @@ namespace ceph { public: Seq1(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; - std::unique_ptr _next(); + std::unique_ptr _next() override; private: int count; @@ -121,6 +128,7 @@ namespace ceph { public: Seq2(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; @@ -133,6 +141,7 @@ namespace ceph { public: Seq3(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; private: @@ -144,6 +153,7 @@ namespace ceph { public: Seq4(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; @@ -156,6 +166,7 @@ namespace ceph { public: Seq5(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; @@ -170,6 +181,7 @@ namespace ceph { public: Seq6(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; @@ -184,6 +196,7 @@ namespace ceph { public: Seq7(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; @@ -197,6 +210,7 @@ namespace ceph { public: Seq8(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; std::unique_ptr _next() override; private: @@ -216,8 +230,8 @@ namespace ceph { public: Seq9(std::pair obj_size_range, int seed); + Sequence get_id() const override; std::string get_name() const override; - std::unique_ptr _next() override; }; } diff --git a/src/common/io_exerciser/JsonStructures.cc b/src/common/io_exerciser/JsonStructures.cc index 12d792a0cbb..ba41c226264 100644 --- a/src/common/io_exerciser/JsonStructures.cc +++ b/src/common/io_exerciser/JsonStructures.cc @@ -1,6 +1,7 @@ #include "JsonStructures.h" #include "common/ceph_json.h" +#include "OpType.h" using namespace ceph::io_exerciser::json; @@ -93,9 +94,147 @@ void OSDMapReply::dump() const formatter->close_section(); } -OSDECProfileSetRequest::OSDECProfileSetRequest(const std::string& name, - std::vector profile, - std::shared_ptr formatter) : +ceph::io_exerciser::json::OSDPoolGetRequest + ::OSDPoolGetRequest(const std::string& pool_name, + std::shared_ptr formatter) : + JSONStructure(formatter), + pool(pool_name) +{ + +} + +ceph::io_exerciser::json::OSDPoolGetRequest + ::OSDPoolGetRequest(JSONObj* obj, + std::shared_ptr formatter) : + JSONStructure(formatter) +{ + ceph::io_exerciser::json::OSDPoolGetRequest::decode_json(obj); +} + +void ceph::io_exerciser::json::OSDPoolGetRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("var", var, obj); + JSONDecoder::decode_json("format", format, obj); +} + +void ceph::io_exerciser::json::OSDPoolGetRequest::dump() const +{ + formatter->open_object_section("OSDPoolGetRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("pool", pool, formatter.get()); + ::encode_json("var", var, formatter.get()); + ::encode_json("format", format, formatter.get()); + formatter->close_section(); +} + +ceph::io_exerciser::json::OSDPoolGetReply + ::OSDPoolGetReply(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void ceph::io_exerciser::json::OSDPoolGetReply::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("erasure_code_profile", erasure_code_profile, obj); +} + +void ceph::io_exerciser::json::OSDPoolGetReply::dump() const +{ + formatter->open_object_section("OSDPoolGetReply"); + ::encode_json("erasure_code_profile", erasure_code_profile, formatter.get()); + formatter->close_section(); +} + +ceph::io_exerciser::json::OSDECProfileGetRequest + ::OSDECProfileGetRequest(const std::string& profile_name, + std::shared_ptr formatter) : + JSONStructure(formatter), + name(profile_name) +{ + +} + +ceph::io_exerciser::json::OSDECProfileGetRequest + ::OSDECProfileGetRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void ceph::io_exerciser::json::OSDECProfileGetRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("name", name, obj); + JSONDecoder::decode_json("format", format, obj); +} + +void ceph::io_exerciser::json::OSDECProfileGetRequest::dump() const +{ + formatter->open_object_section("OSDECProfileGetRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("name", name, formatter.get()); + ::encode_json("format", format, formatter.get()); + formatter->close_section(); +} + +ceph::io_exerciser::json::OSDECProfileGetReply + ::OSDECProfileGetReply(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void ceph::io_exerciser::json::OSDECProfileGetReply::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("crush-device-class", crush_device_class, obj); + JSONDecoder::decode_json("crush-failure-domain", crush_failure_domain, obj); + JSONDecoder::decode_json("crush-num-failure-domains", + crush_num_failure_domains, + obj); + JSONDecoder::decode_json("crush-osds-per-failure-domain", + crush_osds_per_failure_domain, + obj); + JSONDecoder::decode_json("crush-root", crush_root, obj); + JSONDecoder::decode_json("jerasure-per-chunk-alignment", + jerasure_per_chunk_alignment, + obj); + JSONDecoder::decode_json("k", k, obj); + JSONDecoder::decode_json("m", m, obj); + JSONDecoder::decode_json("plugin", plugin, obj); + JSONDecoder::decode_json("technique", technique, obj); + JSONDecoder::decode_json("w", w, obj); +} + +void ceph::io_exerciser::json::OSDECProfileGetReply::dump() const +{ + formatter->open_object_section("OSDECProfileGetReply"); + ::encode_json("crush-device-class", + crush_device_class, + formatter.get()); + ::encode_json("crush-failure-domain", + crush_failure_domain, + formatter.get()); + ::encode_json("crush-num-failure-domains", + crush_num_failure_domains, + formatter.get()); + ::encode_json("crush-osds-per-failure-domain", + crush_osds_per_failure_domain, + formatter.get()); + ::encode_json("crush-root", crush_root, formatter.get()); + ::encode_json("jerasure-per-chunk-alignment", + jerasure_per_chunk_alignment, + formatter.get()); + ::encode_json("k", k, formatter.get()); + ::encode_json("m", m, formatter.get()); + ::encode_json("plugin", plugin, formatter.get()); + ::encode_json("technique", technique, formatter.get()); + ::encode_json("w", w, formatter.get()); + formatter->close_section(); +} + +ceph::io_exerciser::json::OSDECProfileSetRequest + ::OSDECProfileSetRequest(const std::string& name, + const std::vector& profile, + std::shared_ptr formatter) : JSONStructure(formatter), name(name), profile(profile) @@ -140,7 +279,6 @@ OSDECPoolCreateRequest ::OSDECPoolCreateRequest(std::shared_ptr formatter) : JSONStructure(formatter) { - } void OSDECPoolCreateRequest::decode_json(JSONObj* obj) @@ -178,7 +316,6 @@ OSDSetRequest::OSDSetRequest(const std::string& key, OSDSetRequest::OSDSetRequest(std::shared_ptr formatter) : JSONStructure(formatter) { - } void OSDSetRequest::decode_json(JSONObj* obj) @@ -203,7 +340,7 @@ BalancerOffRequest::BalancerOffRequest(std::shared_ptr formatte } -void BalancerOffRequest::decode_json(JSONObj* obj) +void ceph::io_exerciser::json::BalancerOffRequest::decode_json(JSONObj* obj) { JSONDecoder::decode_json("prefix", prefix, obj); } @@ -222,7 +359,7 @@ BalancerStatusRequest } -void BalancerStatusRequest::decode_json(JSONObj* obj) +void ceph::io_exerciser::json::BalancerStatusRequest::decode_json(JSONObj* obj) { JSONDecoder::decode_json("prefix", prefix, obj); } @@ -332,12 +469,18 @@ InjectECErrorRequest::InjectECErrorRequest(InjectOpType injectOpType, { switch(injectOpType) { - case InjectOpType::Read: + case InjectOpType::ReadEIO: + [[ fallthrough ]]; + case InjectOpType::ReadMissingShard: prefix = "injectecreaderr"; break; - case InjectOpType::Write: + case InjectOpType::WriteFailAndRollback: + [[ fallthrough ]]; + case InjectOpType::WriteOSDAbort: prefix = "injectecwriteerr"; break; + default: + ceph_abort_msg("Invalid OP type to inject"); } } @@ -388,12 +531,18 @@ InjectECClearErrorRequest::InjectECClearErrorRequest(InjectOpType injectOpType, { switch(injectOpType) { - case InjectOpType::Read: + case InjectOpType::ReadEIO: + [[ fallthrough ]]; + case InjectOpType::ReadMissingShard: prefix = "injectecclearreaderr"; break; - case InjectOpType::Write: + case InjectOpType::WriteFailAndRollback: + [[ fallthrough ]]; + case InjectOpType::WriteOSDAbort: prefix = "injectecclearwriteerr"; break; + default: + ceph_abort_msg("Invalid OP type to inject"); } } diff --git a/src/common/io_exerciser/JsonStructures.h b/src/common/io_exerciser/JsonStructures.h index 7e1b295919d..b439845b3ab 100644 --- a/src/common/io_exerciser/JsonStructures.h +++ b/src/common/io_exerciser/JsonStructures.h @@ -4,6 +4,8 @@ #include "include/types.h" +#include "OpType.h" + /* Overview * * class JSONStructure @@ -82,11 +84,72 @@ namespace ceph void dump() const; }; + class OSDPoolGetRequest : public JSONStructure + { + public: + OSDPoolGetRequest(const std::string& pool_name, std::shared_ptr formatter = std::make_shared(false)); + OSDPoolGetRequest(JSONObj* obj, std::shared_ptr formatter = std::make_shared(false)); + + std::string prefix = "osd pool get"; + std::string pool; + std::string var = "erasure_code_profile"; + std::string format = "json"; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class OSDPoolGetReply : public JSONStructure + { + public: + OSDPoolGetReply(std::shared_ptr formatter = std::make_shared(false)); + + std::string erasure_code_profile; + + void decode_json(JSONObj *obj); + void dump() const; + }; + + class OSDECProfileGetRequest : public JSONStructure + { + public: + OSDECProfileGetRequest(const std::string& profile_name, std::shared_ptr formatter = std::make_shared(false)); + OSDECProfileGetRequest(std::shared_ptr formatter = std::make_shared(false)); + + std::string prefix = "osd pool get"; + std::string name; + std::string format = "json"; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class OSDECProfileGetReply : public JSONStructure + { + public: + OSDECProfileGetReply(std::shared_ptr formatter = std::make_shared(false)); + + std::string crush_device_class; + std::string crush_failure_domain; + int crush_num_failure_domains; + int crush_osds_per_failure_domain; + std::string crush_root; + bool jerasure_per_chunk_alignment; + int k; + int m; + std::string plugin; + std::string technique; + std::string w; + + void decode_json(JSONObj *obj); + void dump() const; + }; + class OSDECProfileSetRequest : public JSONStructure { public: OSDECProfileSetRequest(const std::string& name, - std::vector profile, + const std::vector& profile, std::shared_ptr formatter = std::make_shared(false)); OSDECProfileSetRequest(std::shared_ptr formatter @@ -203,12 +266,6 @@ namespace ceph void dump() const override; }; - enum class InjectOpType - { - Read, - Write - }; - class InjectECErrorRequest : public JSONStructure { public: diff --git a/src/common/io_exerciser/OpType.h b/src/common/io_exerciser/OpType.h index 737973e0455..84901b372c2 100644 --- a/src/common/io_exerciser/OpType.h +++ b/src/common/io_exerciser/OpType.h @@ -35,6 +35,14 @@ namespace ceph ClearReadErrorInject, // Op to tell OSD to clear read error injects ClearWriteErrorInject // Op to tell OSD to clear write error injects }; + + enum class InjectOpType { + None, + ReadEIO, + ReadMissingShard, + WriteFailAndRollback, + WriteOSDAbort + }; } } diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc index be91b3c27ca..03cb2d26d3f 100644 --- a/src/common/io_exerciser/RadosIo.cc +++ b/src/common/io_exerciser/RadosIo.cc @@ -340,6 +340,7 @@ void RadosIo::applyInjectOp(IoOp& op) auto formatter = std::make_shared(false); int osd = -1; + std::vector shard_order; ceph::io_exerciser::json::OSDMapRequest osdMapRequest(pool, get_oid(), @@ -359,6 +360,9 @@ void RadosIo::applyInjectOp(IoOp& op) reply.decode_json(&p); osd = reply.acting_primary; + shard_order = reply.acting; + + InjectOpType injectOpType; switch(op.getOpType()) { @@ -366,7 +370,20 @@ void RadosIo::applyInjectOp(IoOp& op) { InjectReadErrorOp& errorOp = static_cast(op); - ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(json::InjectOpType::Read, + if (errorOp.type == 0) + { + injectOpType = InjectOpType::ReadEIO; + } + else if (errorOp.type == 1) + { + injectOpType = InjectOpType::ReadMissingShard; + } + else + { + ceph_abort_msg("Unsupported inject type"); + } + + ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(injectOpType, pool, oid, errorOp.shard, @@ -383,7 +400,23 @@ void RadosIo::applyInjectOp(IoOp& op) { InjectWriteErrorOp& errorOp = static_cast(op); - ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(json::InjectOpType::Write, + if (errorOp.type == 0) + { + injectOpType = InjectOpType::WriteFailAndRollback; + } + else if (errorOp.type == 3) + { + injectOpType = InjectOpType::WriteOSDAbort; + + // This inject is sent directly to the shard we want to inject the error on + osd = shard_order[errorOp.shard]; + } + else + { + ceph_abort("Unsupported inject type"); + } + + ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(injectOpType, pool, oid, errorOp.shard, @@ -400,7 +433,20 @@ void RadosIo::applyInjectOp(IoOp& op) { ClearReadErrorInjectOp& errorOp = static_cast(op); - ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(json::InjectOpType::Read, + if (errorOp.type == 0) + { + injectOpType = InjectOpType::ReadEIO; + } + else if (errorOp.type == 1) + { + injectOpType = InjectOpType::ReadMissingShard; + } + else + { + ceph_abort("Unsupported inject type"); + } + + ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(injectOpType, pool, oid, errorOp.shard, @@ -414,7 +460,20 @@ void RadosIo::applyInjectOp(IoOp& op) { ClearReadErrorInjectOp& errorOp = static_cast(op); - ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(json::InjectOpType::Write, + if (errorOp.type == 0) + { + injectOpType = InjectOpType::WriteFailAndRollback; + } + else if (errorOp.type == 3) + { + injectOpType = InjectOpType::WriteOSDAbort; + } + else + { + ceph_abort("Unsupported inject type"); + } + + ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(injectOpType, pool, oid, errorOp.shard, diff --git a/src/test/osd/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence.cc index 0a45739a43d..18852b6aa31 100644 --- a/src/test/osd/ceph_test_rados_io_sequence.cc +++ b/src/test/osd/ceph_test_rados_io_sequence.cc @@ -26,6 +26,7 @@ #include "common/io_exerciser/RadosIo.h" #include "common/io_exerciser/IoOp.h" #include "common/io_exerciser/IoSequence.h" +#include "common/io_exerciser/EcIoSequence.h" #include "common/io_exerciser/JsonStructures.h" #include "json_spirit/json_spirit.h" @@ -186,6 +187,8 @@ namespace { "number of threads of I/O per object (default 1)") ("parallel,p", po::value()->default_value(1), "number of objects to exercise in parallel") + ("testrecovery", + "Inject errors during sequences to test recovery processes of OSDs") ("interactive", "interactive mode, execute IO commands from stdin") ("allow_pool_autoscaling", @@ -330,7 +333,9 @@ const std::pair if (force_value.has_value()) { return *force_value; - } else { + } + else + { return std::make_pair(ceph::io_exerciser::Sequence::SEQUENCE_BEGIN, ceph::io_exerciser::Sequence::SEQUENCE_END); } @@ -396,12 +401,38 @@ const std::string ceph::io_sequence::tester::SelectECPool::choose() { std::pair value; if (!skm.isForced() && force_value.has_value()) { + int rc; + bufferlist inbl, outbl; + auto formatter = std::make_shared(false); + + ceph::io_exerciser::json::OSDPoolGetRequest osdPoolGetRequest(*force_value, formatter); + rc = rados.mon_command(osdPoolGetRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::io_exerciser::json::OSDPoolGetReply osdPoolGetReply(formatter); + osdPoolGetReply.decode_json(&p); + + ceph::io_exerciser::json::OSDECProfileGetRequest osdECProfileGetRequest(osdPoolGetReply.erasure_code_profile, formatter); + rc = rados.mon_command(osdECProfileGetRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + + success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::io_exerciser::json::OSDECProfileGetReply reply(formatter); + reply.decode_json(&p); + k = reply.k; + m = reply.m; return *force_value; } else { value = skm.choose(); } - int k = value.first; - int m = value.second; + k = value.first; + m = value.second; const std::string plugin = std::string(spl.choose()); const uint64_t chunk_size = scs.choose(); @@ -529,16 +560,19 @@ ceph::io_sequence::tester::TestObject::TestObject( const std::string oid, ceph::condition_variable& cond, bool dryrun, bool verbose, - std::optional seqseed) : - rng(rng), verbose(verbose), seqseed(seqseed) + std::optional seqseed, + bool testrecovery) : + rng(rng), verbose(verbose), seqseed(seqseed), testrecovery(testrecovery) { if (dryrun) { - verbose = true; exerciser_model = std::make_unique(oid, sbs.choose(), rng()); } else { const std::string pool = spo.choose(); + poolK = spo.getChosenK(); + poolM = spo.getChosenM(); + int threads = snt.choose(); bufferlist inbl, outbl; @@ -581,14 +615,27 @@ ceph::io_sequence::tester::TestObject::TestObject( const std::string oid, obj_size_range = sos.choose(); seq_range = ssr.choose(); curseq = seq_range.first; - seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq, - obj_size_range, - seqseed.value_or(rng())); + + if (testrecovery) + { + seq = ceph::io_exerciser::EcIoSequence::generate_sequence(curseq, + obj_size_range, + poolK, + poolM, + seqseed.value_or(rng())); + } + else + { + seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq, + obj_size_range, + seqseed.value_or(rng())); + } + op = seq->next(); done = false; dout(0) << "== " << exerciser_model->get_oid() << " " << curseq << " " - << seq->get_name() + << seq->get_name_with_seqseed() << " ==" <applyIoOp(*op); if (op->getOpType() == ceph::io_exerciser::OpType::Done) { - ++curseq; - if (curseq == seq_range.second) + curseq = seq->getNextSupportedSequenceId(); + if (curseq >= seq_range.second) { done = true; dout(0) << exerciser_model->get_oid() @@ -626,11 +673,22 @@ bool ceph::io_sequence::tester::TestObject::next() } else { - seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq, - obj_size_range, - seqseed.value_or(rng())); + if (testrecovery) + { + seq = ceph::io_exerciser::EcIoSequence::generate_sequence(curseq, + obj_size_range, + poolK, poolM, + seqseed.value_or(rng())); + } + else + { + seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq, + obj_size_range, + seqseed.value_or(rng())); + } + dout(0) << "== " << exerciser_model->get_oid() << " " - << curseq << " " << seq->get_name() + << curseq << " " << seq->get_name_with_seqseed() << " ==" <next(); } @@ -681,6 +739,7 @@ ceph::io_sequence::tester::TestRunner::TestRunner(po::variables_map& vm, num_objects = vm["parallel"].as(); object_name = vm["object"].as(); interactive = vm.contains("interactive"); + testrecovery = vm.contains("testrecovery"); allow_pool_autoscaling = vm.contains("allow_pool_autoscaling"); allow_pool_balancer = vm.contains("allow_pool_balancer"); @@ -716,20 +775,30 @@ void ceph::io_sequence::tester::TestRunner::help() } } -void ceph::io_sequence::tester::TestRunner::list_sequence() +void ceph::io_sequence::tester::TestRunner::list_sequence(bool testrecovery) { // List seqeunces std::pair obj_size_range = sos.choose(); - for (ceph::io_exerciser::Sequence s - = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN; - s < ceph::io_exerciser::Sequence::SEQUENCE_END; ++s) + ceph::io_exerciser::Sequence s = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN; + std::unique_ptr seq; + if (testrecovery) { - std::unique_ptr seq = - ceph::io_exerciser::IoSequence::generate_sequence(s, - obj_size_range, - seqseed.value_or(rng())); - dout(0) << s << " " << seq->get_name() << dendl; + seq = ceph::io_exerciser::EcIoSequence::generate_sequence(s, obj_size_range, + spo.getChosenK(), + spo.getChosenM(), + seqseed.value_or(rng())); } + else + { + seq = ceph::io_exerciser::IoSequence::generate_sequence(s, obj_size_range, + seqseed.value_or(rng())); + } + + do + { + dout(0) << s << " " << seq->get_name_with_seqseed() << dendl; + s = seq->getNextSupportedSequenceId(); + } while (s != ceph::io_exerciser::Sequence::SEQUENCE_END); } void ceph::io_sequence::tester::TestRunner::clear_tokens() @@ -801,7 +870,7 @@ bool ceph::io_sequence::tester::TestRunner::run_test() } else if (show_sequence) { - list_sequence(); + list_sequence(testrecovery); return true; } else if (interactive) @@ -1041,7 +1110,7 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() sbs, spo, sos, snt, ssr, rng, lock, cond, dryrun, verbose, - seqseed + seqseed, testrecovery ) ); } diff --git a/src/test/osd/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence.h index e34fdf32d0b..265fa3e23a4 100644 --- a/src/test/osd/ceph_test_rados_io_sequence.h +++ b/src/test/osd/ceph_test_rados_io_sequence.h @@ -251,6 +251,8 @@ namespace ceph bool get_allow_pool_balancer() { return allow_pool_balancer; } bool get_allow_pool_deep_scrubbing() { return allow_pool_deep_scrubbing; } bool get_allow_pool_scrubbing() { return allow_pool_scrubbing; } + int getChosenK() const { return k; } + int getChosenM() const { return m; } private: void create_pool(librados::Rados& rados, @@ -266,7 +268,9 @@ namespace ceph bool allow_pool_balancer; bool allow_pool_deep_scrubbing; bool allow_pool_scrubbing; - + int k; + int m; + SelectErasureKM skm; SelectErasurePlugin spl; SelectErasureChunkSize scs; @@ -288,7 +292,8 @@ namespace ceph ceph::condition_variable& cond, bool dryrun, bool verbose, - std::optional seqseed); + std::optional seqseed, + bool testRecovery); int get_num_io(); bool readyForIo(); @@ -307,6 +312,9 @@ namespace ceph ceph::util::random_number_generator& rng; bool verbose; std::optional seqseed; + int poolK; + int poolM; + bool testrecovery; }; class TestRunner @@ -342,6 +350,8 @@ namespace ceph std::optional seqseed; bool interactive; + bool testrecovery; + bool allow_pool_autoscaling; bool allow_pool_balancer; bool allow_pool_deep_scrubbing; @@ -368,7 +378,7 @@ namespace ceph bool run_interactive_test(); void help(); - void list_sequence(); + void list_sequence(bool testrecovery); }; } } -- 2.39.5