test/common: rados io sequencer exerciser extensions

1. Add miscompare message containing object ID.
2. Fix compiler warning due to strangely placed assert.
3. Add barriers following error injects to insure they
   are in place before IO.
4. Do not, by default, test EC profiles that are not
   known to be good for EC optimisations.
5. Add "allow_unstable_pool_configs" to override above.

Signed-off-by: Alex Ainscow <aainscow@uk.ibm.com>

diff --git a/src/common/io_exerciser/DataGenerator.cc b/src/common/io_exerciser/DataGenerator.cc
index 20acb03bb1bbc3cd97b62ebb7880d6401ce610f0..e91b1df307479ef7943d53c7f5afbf9f7fa698be 100644 (file)
--- a/src/common/io_exerciser/DataGenerator.cc
+++ b/src/common/io_exerciser/DataGenerator.cc
@@ -226,6 +226,8 @@ bool HeaderedSeededRandomGenerator::validate(bufferlist& bufferlist,
   }
 
   if (!invalid_block_offsets.empty()) {
+    dout(0) << "Miscompare for read of " << m_model.get_oid() <<
+      " offset=" << offset << " length=" << length << dendl;
     printDebugInformationForOffsets(offset, invalid_block_offsets, bufferlist);
   }
 

diff --git a/src/common/io_exerciser/EcIoSequence.cc b/src/common/io_exerciser/EcIoSequence.cc
index 4b879fc7d5b3cd20a26db4b0454c95d7f587717b..1878f2f7002f87de922744388f47168a725f02ef 100644 (file)
--- a/src/common/io_exerciser/EcIoSequence.cc
+++ b/src/common/io_exerciser/EcIoSequence.cc
@@ -207,8 +207,8 @@ std::unique_ptr<IoOp> ReadInjectSequence::next() {
   switch (child_op->getOpType()) {
     case OpType::Remove:
       next_op.swap(child_op);
+      ceph_assert(shard_to_inject.has_value());
       switch (inject_op_type) {
-        ceph_assert(shard_to_inject.has_value());
         case InjectOpType::ReadEIO:
           return ClearReadErrorInjectOp::generate(*shard_to_inject, 0);
         case InjectOpType::ReadMissingShard:
@@ -303,6 +303,7 @@ std::string ceph::io_exerciser::Seq10::get_name() const {
 std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq10::_next() {
   if (!inject_error_done) {
     inject_error_done = true;
+    barrier = true;
     return InjectWriteErrorOp::generate(*shard_to_inject, 0, 0,
                                         std::numeric_limits<uint64_t>::max());
   } else if (!failed_write_done) {
@@ -316,6 +317,7 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq10::_next() {
     return SingleReadOp::generate(offset, length);
   } else if (!clear_inject_done) {
     clear_inject_done = true;
+    barrier = true;
     return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0);
   } else if (!successful_write_done) {
     successful_write_done = true;

diff --git a/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h b/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h
index 12d926aefdc50613b2069307dcb5be18c06deb86..cf56134d447c7d09b0c6b4789714cb4d2558dff5 100644 (file)
--- a/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h
+++ b/src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h
@@ -115,6 +115,53 @@ class ProgramOptionSelector : public ProgramOptionReader<option_type> {
   std::optional<option_type> first_value;
 };
 
+template <typename option_type,
+          int num_selections,
+          const std::array< option_type,
+                            num_selections>& selections_array,
+          int num_selections_stable,
+          const std::array< option_type,
+                            num_selections_stable>& elections_array_stable>
+class StableOptionSelector : public ProgramOptionReader<option_type> {
+public:
+  StableOptionSelector(ceph::util::random_number_generator<int>& rng,
+                        po::variables_map& vm,
+                        const std::string& option_name,
+                        bool select_first)
+      : ProgramOptionReader<option_type>(vm, option_name), rng(rng),
+        stable(!vm.contains("allow_unstable_pool_configs") ||
+          vm.contains("disable_pool_ec_optimizations")) {
+    if (select_first) {
+      if (stable) {
+        ceph_assert(selections_array.size() > 0);
+        first_value = elections_array_stable[0];
+      } else {
+        ceph_assert(selections_array.size() > 0);
+        first_value = selections_array[0];
+      }
+    }
+  }
+
+  virtual ~StableOptionSelector() = default;
+
+  virtual const option_type select() override {
+    if (this->force_value.has_value()) {
+      return *this->force_value;
+    } else if (first_value.has_value()) {
+      return *std::exchange(first_value, std::nullopt);
+    } else if (stable) {
+      return elections_array_stable[rng(num_selections_stable - 1)];
+    } else {
+      return selections_array[rng(num_selections - 1)];
+    }
+  }
+
+protected:
+  ceph::util::random_number_generator<int>& rng;
+  std::optional<option_type> first_value;
+  bool stable;
+};
+
 template <typename option_type>
 class ProgramOptionGeneratedSelector
     : public OptionalProgramOptionReader<option_type> {

diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc
index 2d6806fbed8a573c78b38bf7aeec95583d93b1bb..77049721af84dd5a096f197ba2422a8a87d28677 100644 (file)
--- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc
+++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc
@@ -184,7 +184,12 @@ po::options_description get_options_description() {
       "allow_pool_balancer", "Enables pool balancing. Disabled by default.")(
       "allow_pool_deep_scrubbing",
       "Enables pool deep scrub. Disabled by default.")(
-      "allow_pool_scrubbing", "Enables pool scrubbing. Disabled by default.");
+      "allow_pool_scrubbing", "Enables pool scrubbing. Disabled by default.")(
+      "disable_pool_ec_optimizations",
+      "Disables EC optimizations. Enabled by default.")(
+      "allow_unstable_pool_configs",
+      "Permits pool configs that are known to be unstable. This option "
+      " may be removed. at a later date. Disabled by default if ec optimized");
 
   return desc;
 }
@@ -272,22 +277,28 @@ ceph::io_sequence::tester::SelectErasureTechnique::SelectErasureTechnique(
     : ProgramOptionGeneratedSelector<std::string>(rng, vm, "technique",
                                                   first_use),
       rng(rng),
-      plugin(plugin) {}
+      plugin(plugin),
+      stable(!vm.contains("allow_unstable_pool_configs") ||
+        vm.contains("disable_pool_ec_optimizations")) {}
 
 const std::vector<std::string>
 ceph::io_sequence::tester::SelectErasureTechnique::generate_selections() {
   std::vector<std::string> techniques = {};
   if (plugin == "jerasure") {
     techniques.push_back("reed_sol_van");
-    techniques.push_back("reed_sol_r6_op");
-    techniques.push_back("cauchy_orig");
-    techniques.push_back("cauchy_good");
-    techniques.push_back("liberation");
-    techniques.push_back("blaum_roth");
-    techniques.push_back("liber8tion");
+    if (!stable) {
+      techniques.push_back("reed_sol_r6_op");
+      techniques.push_back("cauchy_orig");
+      techniques.push_back("cauchy_good");
+      techniques.push_back("liberation");
+      techniques.push_back("blaum_roth");
+      techniques.push_back("liber8tion");
+    }
   } else if (plugin == "isa") {
     techniques.push_back("reed_sol_van");
-    techniques.push_back("cauchy");
+    if (!stable) {
+      techniques.push_back("cauchy");
+    }
   } else if (plugin == "shec") {
     techniques.push_back("single");
     techniques.push_back("multiple");
@@ -337,28 +348,24 @@ ceph::io_sequence::tester::SelectErasureKM::generate_selections() {
        (technique == "reed_sol_van" || technique == "cauchy_orig" ||
         technique == "cauchy_good" || technique == std::nullopt))) {
     for (int m = 1; m <= 3; m++)
-      for (int k = 2; k <= 6; k++) selection.push_back({k, m});
+      for (int k = 2; k <= 4; k++) selection.push_back({k, m});
   } else if (plugin == "shec" ||
              (plugin == "jerasure" &&
               (technique == "liberation" || technique == "blaum_roth"))) {
     for (int m = 1; m <= 2; m++)
-      for (int k = 2; k <= 6; k++) selection.push_back({k, m});
+      for (int k = 2; k <= 4; k++) selection.push_back({k, m});
   } else if (plugin == "jerasure" &&
              (technique == "reed_sol_r6_op" || technique == "liber8tion")) {
-    for (int k = 2; k <= 6; k++) selection.push_back({k, 2});
+    for (int k = 2; k <= 4; k++) selection.push_back({k, 2});
   }
 
   // We want increased chances of these as we will test with c=1 and c=2
   if (plugin == "shec")
     for (int i = 0; i < 2; i++)
-      for (int k = 3; k <= 6; k++) selection.push_back({k, 3});
+      for (int k = 3; k <= 4; k++) selection.push_back({k, 3});
 
   // Add extra miscelaneous interesting options for testing w values
   if (plugin == "jerasure") {
-    if (technique == "reed_sol_van")
-      // Double chance of chosing to test more w values
-      for (int i = 0; i < 2; i++) selection.push_back({6, 3});
-
     if (technique == "liberation" || technique == "blaum_roth")
       // Double chance of chosing to test more different w values
       for (int i = 0; i < 2; i++) selection.push_back({6, 2});

diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h
index dabe8faf748bb04473050084fe38b02864255142..eeb3abb6cedd13fb8113c116fdc81d5dcde06e1a 100644 (file)
--- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h
+++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h
@@ -108,10 +108,19 @@ inline static constexpr std::array<uint64_t, block_size_array_size> block_size_c
     {2048,  // Default - test boundaries for EC 4K chunk size
      512, 3767, 4096, 32768}};
 
+// Choices for block size
+inline static constexpr int block_size_array_size_stable = 2;
+inline static constexpr std::array<uint64_t, block_size_array_size_stable> block_size_choices_stable = {
+  {2048,  // Default - test boundaries for EC 4K chunk size
+   32768}};
+
+
 using SelectBlockSize =
-    ProgramOptionSelector<uint64_t,
-                          io_sequence::tester::block_size_array_size,
-                          io_sequence::tester::block_size_choices>;
+  StableOptionSelector<uint64_t,
+                        io_sequence::tester::block_size_array_size,
+                        io_sequence::tester::block_size_choices,
+                        io_sequence::tester::block_size_array_size_stable,
+                        io_sequence::tester::block_size_choices_stable>;
 
 // Choices for number of threads
 inline static constexpr int thread_array_size = 4;
@@ -138,10 +147,16 @@ inline static constexpr int plugin_array_size = 5;
 inline static constexpr std::array<std::string_view, plugin_array_size>
     plugin_choices = {{"jerasure", "isa", "clay", "shec", "lrc"}};
 
+inline static constexpr int plugin_array_size_stable = 2;
+inline static constexpr std::array<std::string_view, plugin_array_size_stable>
+    plugin_choices_stable = {{"jerasure", "isa"}};
+
 using SelectErasurePlugin =
-    ProgramOptionSelector<std::string_view,
+    StableOptionSelector<std::string_view,
                           io_sequence::tester::plugin_array_size,
-                          io_sequence::tester::plugin_choices>;
+                          io_sequence::tester::plugin_choices,
+                          io_sequence::tester::plugin_array_size_stable,
+                          io_sequence::tester::plugin_choices_stable>;
 
 class SelectErasureKM
     : public ProgramOptionGeneratedSelector<std::pair<int, int>> {
@@ -306,6 +321,7 @@ class SelectErasureTechnique
   ceph::util::random_number_generator<int>& rng;
 
   std::string_view plugin;
+  bool stable;
 };
 
 class SelectErasureChunkSize : public ProgramOptionGeneratedSelector<uint64_t> {