From 96dce17e51ba6425682e56ed67f20576b0b72f9d Mon Sep 17 00:00:00 2001 From: Alex Ainscow Date: Thu, 27 Mar 2025 11:45:33 +0000 Subject: [PATCH] test/common: rados io sequencer exerciser extensions 1. Add miscompare message containing object ID. 2. Fix compiler warning due to strangely placed assert. 3. Add barriers following error injects to insure they are in place before IO. 4. Do not, by default, test EC profiles that are not known to be good for EC optimisations. 5. Add "allow_unstable_pool_configs" to override above. Signed-off-by: Alex Ainscow --- src/common/io_exerciser/DataGenerator.cc | 2 + src/common/io_exerciser/EcIoSequence.cc | 4 +- .../ProgramOptionReader.h | 47 +++++++++++++++++++ .../ceph_test_rados_io_sequence.cc | 41 +++++++++------- .../ceph_test_rados_io_sequence.h | 26 ++++++++-- 5 files changed, 97 insertions(+), 23 deletions(-) diff --git a/src/common/io_exerciser/DataGenerator.cc b/src/common/io_exerciser/DataGenerator.cc index 20acb03bb1bb..e91b1df30747 100644 --- a/src/common/io_exerciser/DataGenerator.cc +++ b/src/common/io_exerciser/DataGenerator.cc @@ -226,6 +226,8 @@ bool HeaderedSeededRandomGenerator::validate(bufferlist& bufferlist, } if (!invalid_block_offsets.empty()) { + dout(0) << "Miscompare for read of " << m_model.get_oid() << + " offset=" << offset << " length=" << length << dendl; printDebugInformationForOffsets(offset, invalid_block_offsets, bufferlist); } diff --git a/src/common/io_exerciser/EcIoSequence.cc b/src/common/io_exerciser/EcIoSequence.cc index 4b879fc7d5b3..1878f2f7002f 100644 --- a/src/common/io_exerciser/EcIoSequence.cc +++ b/src/common/io_exerciser/EcIoSequence.cc @@ -207,8 +207,8 @@ std::unique_ptr ReadInjectSequence::next() { switch (child_op->getOpType()) { case OpType::Remove: next_op.swap(child_op); + ceph_assert(shard_to_inject.has_value()); switch (inject_op_type) { - ceph_assert(shard_to_inject.has_value()); case InjectOpType::ReadEIO: return ClearReadErrorInjectOp::generate(*shard_to_inject, 0); case InjectOpType::ReadMissingShard: @@ -303,6 +303,7 @@ std::string ceph::io_exerciser::Seq10::get_name() const { std::unique_ptr ceph::io_exerciser::Seq10::_next() { if (!inject_error_done) { inject_error_done = true; + barrier = true; return InjectWriteErrorOp::generate(*shard_to_inject, 0, 0, std::numeric_limits::max()); } else if (!failed_write_done) { @@ -316,6 +317,7 @@ std::unique_ptr ceph::io_exerciser::Seq10::_next() { return SingleReadOp::generate(offset, length); } else if (!clear_inject_done) { clear_inject_done = true; + barrier = true; return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0); } else if (!successful_write_done) { successful_write_done = true; diff --git a/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h b/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h index 12d926aefdc5..cf56134d447c 100644 --- a/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h +++ b/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h @@ -115,6 +115,53 @@ class ProgramOptionSelector : public ProgramOptionReader { std::optional first_value; }; +template & selections_array, + int num_selections_stable, + const std::array< option_type, + num_selections_stable>& elections_array_stable> +class StableOptionSelector : public ProgramOptionReader { +public: + StableOptionSelector(ceph::util::random_number_generator& rng, + po::variables_map& vm, + const std::string& option_name, + bool select_first) + : ProgramOptionReader(vm, option_name), rng(rng), + stable(!vm.contains("allow_unstable_pool_configs") || + vm.contains("disable_pool_ec_optimizations")) { + if (select_first) { + if (stable) { + ceph_assert(selections_array.size() > 0); + first_value = elections_array_stable[0]; + } else { + ceph_assert(selections_array.size() > 0); + first_value = selections_array[0]; + } + } + } + + virtual ~StableOptionSelector() = default; + + virtual const option_type select() override { + if (this->force_value.has_value()) { + return *this->force_value; + } else if (first_value.has_value()) { + return *std::exchange(first_value, std::nullopt); + } else if (stable) { + return elections_array_stable[rng(num_selections_stable - 1)]; + } else { + return selections_array[rng(num_selections - 1)]; + } + } + +protected: + ceph::util::random_number_generator& rng; + std::optional first_value; + bool stable; +}; + template class ProgramOptionGeneratedSelector : public OptionalProgramOptionReader { diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc index 2d6806fbed8a..77049721af84 100644 --- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc +++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc @@ -184,7 +184,12 @@ po::options_description get_options_description() { "allow_pool_balancer", "Enables pool balancing. Disabled by default.")( "allow_pool_deep_scrubbing", "Enables pool deep scrub. Disabled by default.")( - "allow_pool_scrubbing", "Enables pool scrubbing. Disabled by default."); + "allow_pool_scrubbing", "Enables pool scrubbing. Disabled by default.")( + "disable_pool_ec_optimizations", + "Disables EC optimizations. Enabled by default.")( + "allow_unstable_pool_configs", + "Permits pool configs that are known to be unstable. This option " + " may be removed. at a later date. Disabled by default if ec optimized"); return desc; } @@ -272,22 +277,28 @@ ceph::io_sequence::tester::SelectErasureTechnique::SelectErasureTechnique( : ProgramOptionGeneratedSelector(rng, vm, "technique", first_use), rng(rng), - plugin(plugin) {} + plugin(plugin), + stable(!vm.contains("allow_unstable_pool_configs") || + vm.contains("disable_pool_ec_optimizations")) {} const std::vector ceph::io_sequence::tester::SelectErasureTechnique::generate_selections() { std::vector techniques = {}; if (plugin == "jerasure") { techniques.push_back("reed_sol_van"); - techniques.push_back("reed_sol_r6_op"); - techniques.push_back("cauchy_orig"); - techniques.push_back("cauchy_good"); - techniques.push_back("liberation"); - techniques.push_back("blaum_roth"); - techniques.push_back("liber8tion"); + if (!stable) { + techniques.push_back("reed_sol_r6_op"); + techniques.push_back("cauchy_orig"); + techniques.push_back("cauchy_good"); + techniques.push_back("liberation"); + techniques.push_back("blaum_roth"); + techniques.push_back("liber8tion"); + } } else if (plugin == "isa") { techniques.push_back("reed_sol_van"); - techniques.push_back("cauchy"); + if (!stable) { + techniques.push_back("cauchy"); + } } else if (plugin == "shec") { techniques.push_back("single"); techniques.push_back("multiple"); @@ -337,28 +348,24 @@ ceph::io_sequence::tester::SelectErasureKM::generate_selections() { (technique == "reed_sol_van" || technique == "cauchy_orig" || technique == "cauchy_good" || technique == std::nullopt))) { for (int m = 1; m <= 3; m++) - for (int k = 2; k <= 6; k++) selection.push_back({k, m}); + for (int k = 2; k <= 4; k++) selection.push_back({k, m}); } else if (plugin == "shec" || (plugin == "jerasure" && (technique == "liberation" || technique == "blaum_roth"))) { for (int m = 1; m <= 2; m++) - for (int k = 2; k <= 6; k++) selection.push_back({k, m}); + for (int k = 2; k <= 4; k++) selection.push_back({k, m}); } else if (plugin == "jerasure" && (technique == "reed_sol_r6_op" || technique == "liber8tion")) { - for (int k = 2; k <= 6; k++) selection.push_back({k, 2}); + for (int k = 2; k <= 4; k++) selection.push_back({k, 2}); } // We want increased chances of these as we will test with c=1 and c=2 if (plugin == "shec") for (int i = 0; i < 2; i++) - for (int k = 3; k <= 6; k++) selection.push_back({k, 3}); + for (int k = 3; k <= 4; k++) selection.push_back({k, 3}); // Add extra miscelaneous interesting options for testing w values if (plugin == "jerasure") { - if (technique == "reed_sol_van") - // Double chance of chosing to test more w values - for (int i = 0; i < 2; i++) selection.push_back({6, 3}); - if (technique == "liberation" || technique == "blaum_roth") // Double chance of chosing to test more different w values for (int i = 0; i < 2; i++) selection.push_back({6, 2}); diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h index dabe8faf748b..eeb3abb6cedd 100644 --- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h +++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h @@ -108,10 +108,19 @@ inline static constexpr std::array block_size_c {2048, // Default - test boundaries for EC 4K chunk size 512, 3767, 4096, 32768}}; +// Choices for block size +inline static constexpr int block_size_array_size_stable = 2; +inline static constexpr std::array block_size_choices_stable = { + {2048, // Default - test boundaries for EC 4K chunk size + 32768}}; + + using SelectBlockSize = - ProgramOptionSelector; + StableOptionSelector; // Choices for number of threads inline static constexpr int thread_array_size = 4; @@ -138,10 +147,16 @@ inline static constexpr int plugin_array_size = 5; inline static constexpr std::array plugin_choices = {{"jerasure", "isa", "clay", "shec", "lrc"}}; +inline static constexpr int plugin_array_size_stable = 2; +inline static constexpr std::array + plugin_choices_stable = {{"jerasure", "isa"}}; + using SelectErasurePlugin = - ProgramOptionSelector; + io_sequence::tester::plugin_choices, + io_sequence::tester::plugin_array_size_stable, + io_sequence::tester::plugin_choices_stable>; class SelectErasureKM : public ProgramOptionGeneratedSelector> { @@ -306,6 +321,7 @@ class SelectErasureTechnique ceph::util::random_number_generator& rng; std::string_view plugin; + bool stable; }; class SelectErasureChunkSize : public ProgramOptionGeneratedSelector { -- 2.47.3