common/io_exerciser: Add simple sequences for testing error injects

Add sequences to test IOs with simple error injects, along with some small fixes for previous error inject implementation.

Signed-off-by: Jon Bailey <jonathan.bailey1@ibm.com>

diff --git a/src/common/io_exerciser/CMakeLists.txt b/src/common/io_exerciser/CMakeLists.txt
index 930cfec9ab561b1f8f7eb681505e100cfc92eef3..ccad61b57a215ad437787862d8a8c2f409f8b91d 100644 (file)
--- a/src/common/io_exerciser/CMakeLists.txt
+++ b/src/common/io_exerciser/CMakeLists.txt
@@ -6,6 +6,7 @@ add_library(object_io_exerciser STATIC
   ObjectModel.cc
   RadosIo.cc
   JsonStructures.cc
+  EcIoSequence.cc
 )
 
 target_link_libraries(object_io_exerciser

diff --git a/src/common/io_exerciser/DataGenerator.h b/src/common/io_exerciser/DataGenerator.h
index 1e5784a54ccd703a0374f01bc87edf0daaabdde3..886454fdfa440194d9e5c2ebff4e978966460716 100644 (file)
--- a/src/common/io_exerciser/DataGenerator.h
+++ b/src/common/io_exerciser/DataGenerator.h
@@ -65,7 +65,7 @@ namespace ceph {
             : DataGenerator(model) {}
 
           virtual bufferptr generate_block(uint64_t offset);
-          virtual bufferlist generate_data(uint64_t length, uint64_t offset);
+          virtual bufferlist generate_data(uint64_t length, uint64_t offset) override;
           virtual bufferptr generate_wrong_block(uint64_t offset);
           virtual bufferlist generate_wrong_data(uint64_t offset, uint64_t length) override;
       };

diff --git a/src/common/io_exerciser/EcIoSequence.cc b/src/common/io_exerciser/EcIoSequence.cc
new file mode 100644 (file)
index 0000000..cc605de
--- /dev/null
+++ b/src/common/io_exerciser/EcIoSequence.cc
@@ -0,0 +1,296 @@
+#include "EcIoSequence.h"
+
+#include <memory>
+
+using IoOp = ceph::io_exerciser::IoOp;
+using Sequence = ceph::io_exerciser::Sequence;
+using IoSequence = ceph::io_exerciser::IoSequence;
+using EcIoSequence = ceph::io_exerciser::EcIoSequence;
+using ReadInjectSequence = ceph::io_exerciser::ReadInjectSequence;
+
+bool EcIoSequence::is_supported(Sequence sequence) const
+{
+  return true;
+}
+
+std::unique_ptr<IoSequence> EcIoSequence::generate_sequence(Sequence sequence,
+                                                            std::pair<int,int> obj_size_range,
+                                                            int k,
+                                                            int m,
+                                                            int seed)
+{
+  switch(sequence)
+  {
+    case Sequence::SEQUENCE_SEQ0:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ1:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ2:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ3:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ4:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ5:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ6:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ7:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ8:
+      [[ fallthrough ]];
+    case Sequence::SEQUENCE_SEQ9:
+      return std::make_unique<ReadInjectSequence>(obj_size_range, seed,
+                                                  sequence, k, m);
+    case Sequence::SEQUENCE_SEQ10:
+      return std::make_unique<Seq10>(obj_size_range, seed, k, m);
+    default:
+      ceph_abort_msg("Unrecognised sequence");
+  }
+}
+
+EcIoSequence::EcIoSequence(std::pair<int,int> obj_size_range, int seed) :
+  IoSequence(obj_size_range, seed),
+  setup_inject(false), clear_inject(false), shard_to_inject(std::nullopt)
+{
+
+}
+
+void EcIoSequence::select_random_data_shard_to_inject_read_error(int k, int m)
+{
+  shard_to_inject = rng(k - 1);
+  setup_inject = true;
+}
+
+void EcIoSequence::select_random_data_shard_to_inject_write_error(int k, int m)
+{
+  // Write errors do not support injecting to the primary OSD
+  shard_to_inject = rng(1, k - 1);
+  setup_inject = true;
+}
+
+void EcIoSequence::select_random_shard_to_inject_read_error(int k, int m)
+{
+  shard_to_inject = rng(k + m - 1);
+  setup_inject = true;
+}
+
+void EcIoSequence::select_random_shard_to_inject_write_error(int k, int m)
+{
+  // Write errors do not support injecting to the primary OSD
+  shard_to_inject = rng(1, k + m - 1);
+  setup_inject = true;
+}
+
+void EcIoSequence::generate_random_read_inject_type()
+{
+  inject_op_type = static_cast<InjectOpType>(rng(static_cast<int>(InjectOpType::ReadEIO),
+                                                 static_cast<int>(InjectOpType::ReadMissingShard)));
+}
+
+void EcIoSequence::generate_random_write_inject_type()
+{
+  inject_op_type = static_cast<InjectOpType>(rng(static_cast<int>(InjectOpType::WriteFailAndRollback),
+                                                 static_cast<int>(InjectOpType::WriteOSDAbort)));
+}
+
+ceph::io_exerciser::ReadInjectSequence::ReadInjectSequence(std::pair<int,int> obj_size_range,
+                                                           int seed,
+                                                           Sequence s,
+                                                           int k, int m) :
+  EcIoSequence(obj_size_range, seed)
+{
+  child_sequence = IoSequence::generate_sequence(s, obj_size_range, seed);
+  select_random_data_shard_to_inject_read_error(k, m);
+  generate_random_read_inject_type();
+}
+
+Sequence ceph::io_exerciser::ReadInjectSequence::get_id() const
+{
+  return child_sequence->get_id();
+}
+
+std::string ceph::io_exerciser::ReadInjectSequence::get_name() const
+{
+  return child_sequence->get_name() +
+    " running with read errors injected on shard "
+    + std::to_string(*shard_to_inject);
+}
+
+std::unique_ptr<IoOp> ReadInjectSequence::next()
+{
+  step++;
+
+  if (nextOp)
+  {
+    std::unique_ptr<IoOp> retOp = nullptr;
+    nextOp.swap(retOp);
+    return retOp;
+  }
+
+  std::unique_ptr<IoOp> childOp = child_sequence->next();
+
+  switch(childOp->getOpType())
+  {
+    case OpType::Remove:
+      nextOp.swap(childOp);
+        switch(inject_op_type)
+        {
+          case InjectOpType::ReadEIO:
+            return ClearReadErrorInjectOp::generate(*shard_to_inject, 0);
+          case InjectOpType::ReadMissingShard:
+            return ClearReadErrorInjectOp::generate(*shard_to_inject, 1);
+          case InjectOpType::WriteFailAndRollback:
+            return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0);
+          case InjectOpType::WriteOSDAbort:
+            return ClearWriteErrorInjectOp::generate(*shard_to_inject, 3);
+          case InjectOpType::None:
+            [[ fallthrough ]];
+          default:
+            ceph_abort_msg("Unsupported operation");
+        }
+      break;
+    case OpType::Create:
+      switch(inject_op_type)
+      {
+        case InjectOpType::ReadEIO:
+          nextOp = InjectReadErrorOp::generate(*shard_to_inject, 0, 0, std::numeric_limits<uint64_t>::max());
+          break;
+        case InjectOpType::ReadMissingShard:
+          nextOp = InjectReadErrorOp::generate(*shard_to_inject, 1, 0, std::numeric_limits<uint64_t>::max());
+          break;
+        case InjectOpType::WriteFailAndRollback:
+          nextOp = InjectWriteErrorOp::generate(*shard_to_inject, 0, 0, std::numeric_limits<uint64_t>::max());
+          break;
+        case InjectOpType::WriteOSDAbort:
+          nextOp = InjectWriteErrorOp::generate(*shard_to_inject, 3, 0, std::numeric_limits<uint64_t>::max());
+          break;
+        case InjectOpType::None:
+          [[ fallthrough ]];
+        default:
+          ceph_abort_msg("Unsupported operation");
+      }
+    break;
+    default:
+      // Do nothing in default case
+      break;
+  }
+
+  return childOp;
+}
+
+std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::ReadInjectSequence::_next()
+{
+  ceph_abort_msg("Should not reach this point, "
+                 "this sequence should only consume complete sequences");
+
+  return DoneOp::generate();
+}
+
+
+
+ceph::io_exerciser::Seq10::Seq10(std::pair<int,int> obj_size_range, int seed,
+                                 int k, int m) :
+  EcIoSequence(obj_size_range, seed), offset(0), length(1),
+  failed_write_done(false), read_done(false), successful_write_done(false),
+  test_all_lengths(false), // Only test length(1) due to time constraints
+  test_all_sizes(false) // Only test obj_size(rand()) due to time constraints
+{
+  select_random_shard_to_inject_write_error(k, m);
+  // We will inject specifically as part of our sequence in this sequence
+  setup_inject = false;
+  if (!test_all_sizes)
+  {
+    select_random_object_size();
+  }
+}
+
+Sequence ceph::io_exerciser::Seq10::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ10;
+}
+
+std::string ceph::io_exerciser::Seq10::get_name() const
+{
+  return "Sequential writes of length " + std::to_string(length) +
+    " with queue depth 1"
+    " first injecting a failed write and read it to ensure it rolls back, then"
+    " successfully writing the data and reading the write the ensure it is applied";
+}
+
+std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq10::_next()
+{
+  if (!inject_error_done)
+  {
+    inject_error_done = true;
+    return InjectWriteErrorOp::generate(*shard_to_inject, 0, 0,
+                                        std::numeric_limits<uint64_t>::max());
+  }
+  else if (!failed_write_done)
+  {
+    failed_write_done = true;
+    read_done = false;
+    barrier = true;
+    return SingleFailedWriteOp::generate(offset, length);
+  }
+  else if (failed_write_done && !read_done)
+  {
+    read_done = true;
+    barrier = true;
+    return SingleReadOp::generate(offset, length);
+  }
+  else if (!clear_inject_done)
+  {
+    clear_inject_done = true;
+    return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0);
+  }
+  else if (!successful_write_done)
+  {
+    successful_write_done = true;
+    read_done = false;
+    barrier = true;
+    return SingleWriteOp::generate(offset, length);
+  }
+  else if (successful_write_done && !read_done)
+  {
+    read_done = true;
+    return SingleReadOp::generate(offset, length);
+  }
+  else if (successful_write_done && read_done)
+  {
+    offset++;
+    inject_error_done = false;
+    failed_write_done = false;
+    read_done = false;
+    clear_inject_done = false;
+    successful_write_done = false;
+
+    if (offset + length >= obj_size) {
+      if (!test_all_lengths)
+      {
+        done = true;
+        return BarrierOp::generate();
+      }
+
+      offset = 0;
+      length++;
+      if (length > obj_size) {
+        if (!test_all_sizes)
+        {
+          done = true;
+          return BarrierOp::generate();
+        }
+
+        length = 1;
+        return increment_object_size();
+      }
+    }
+
+    return BarrierOp::generate();
+  }
+  else
+  {
+    ceph_abort_msg("Sequence in undefined state. Aborting");
+    return DoneOp::generate();
+  }
+}
\ No newline at end of file

diff --git a/src/common/io_exerciser/EcIoSequence.h b/src/common/io_exerciser/EcIoSequence.h
new file mode 100644 (file)
index 0000000..acfbb34
--- /dev/null
+++ b/src/common/io_exerciser/EcIoSequence.h
@@ -0,0 +1,71 @@
+#include "IoSequence.h"
+
+namespace ceph
+{
+  namespace io_exerciser
+  {
+    class EcIoSequence : public IoSequence
+    {
+      public:
+        virtual bool is_supported(Sequence sequence) const override;
+        static std::unique_ptr<IoSequence>
+          generate_sequence(Sequence s,
+                            std::pair<int,int> obj_size_range,
+                            int k,
+                            int m,
+                            int seed );
+
+      protected:
+        bool setup_inject;
+        bool clear_inject;
+        std::optional<uint64_t> shard_to_inject;
+        InjectOpType inject_op_type;
+
+        EcIoSequence(std::pair<int,int> obj_size_range, int seed);
+
+        // Writes cannot be sent to injected on shard zero, so selections seperated out
+        void select_random_data_shard_to_inject_read_error(int k, int m);
+        void select_random_data_shard_to_inject_write_error(int k, int m);
+        void select_random_shard_to_inject_read_error(int k, int m);
+        void select_random_shard_to_inject_write_error(int k, int m);
+        void generate_random_read_inject_type();
+        void generate_random_write_inject_type();
+    };
+
+    class ReadInjectSequence : public EcIoSequence
+    {
+    public:
+      ReadInjectSequence(std::pair<int,int> obj_size_range, int seed, Sequence s, int k, int m);
+
+      Sequence get_id() const override;
+      std::string get_name() const override;
+      virtual std::unique_ptr<IoOp> next() override;
+      std::unique_ptr<IoOp> _next() override;
+
+      private:
+        std::unique_ptr<IoSequence> child_sequence;
+        std::unique_ptr<IoOp> nextOp;
+    };
+
+    class Seq10: public EcIoSequence {
+    public:
+      Seq10(std::pair<int,int> obj_size_range, int seed, int k, int m);
+
+      Sequence get_id() const override;
+      std::string get_name() const override;
+      std::unique_ptr<IoOp> _next() override;
+
+    private:
+      uint64_t offset;
+      uint64_t length;
+
+      bool inject_error_done;
+      bool failed_write_done;
+      bool read_done;
+      bool clear_inject_done;
+      bool successful_write_done;
+      bool test_all_lengths;
+      bool test_all_sizes;
+    };
+  }
+}
\ No newline at end of file

diff --git a/src/common/io_exerciser/IoSequence.cc b/src/common/io_exerciser/IoSequence.cc
index d4b7af40b413747c496b88d0febeeb56bff86f44..f14f0d117da8dc382f4b39e926137f4a3f9d6848 100644 (file)
--- a/src/common/io_exerciser/IoSequence.cc
+++ b/src/common/io_exerciser/IoSequence.cc
@@ -38,6 +38,9 @@ std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& s
     case Sequence::SEQUENCE_SEQ9:
       os << "SEQUENCE_SEQ9";
       break;
+    case Sequence::SEQUENCE_SEQ10:
+      os << "SEQUENCE_SEQ10";
+      break;
     case Sequence::SEQUENCE_END:
       os << "SEQUENCE_END";
       break;
@@ -45,6 +48,11 @@ std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& s
   return os;
 }
 
+bool IoSequence::is_supported(Sequence sequence) const
+{
+  return sequence != Sequence::SEQUENCE_SEQ10;
+}
+
 std::unique_ptr<IoSequence> IoSequence::generate_sequence(Sequence s,
                                                           std::pair<int,int> obj_size_range,
                                                           int seed)
@@ -70,6 +78,10 @@ std::unique_ptr<IoSequence> IoSequence::generate_sequence(Sequence s,
       return std::make_unique<Seq8>(obj_size_range, seed);
     case Sequence::SEQUENCE_SEQ9:
       return std::make_unique<Seq9>(obj_size_range, seed);
+    case Sequence::SEQUENCE_SEQ10:
+      ceph_abort_msg("Sequence 10 only supported for erasure coded pools "
+                     "through the EcIoSequence interface");
+      return nullptr;
     default:
       break;
   }
@@ -77,7 +89,7 @@ std::unique_ptr<IoSequence> IoSequence::generate_sequence(Sequence s,
 }
 
 IoSequence::IoSequence(std::pair<int,int> obj_size_range,
-                                           int seed) :
+                       int seed) :
         min_obj_size(obj_size_range.first), max_obj_size(obj_size_range.second),
         create(true), barrier(false), done(false), remove(false),
         obj_size(min_obj_size), step(-1), seed(seed)
@@ -85,6 +97,11 @@ IoSequence::IoSequence(std::pair<int,int> obj_size_range,
   rng.seed(seed);
 }
 
+std::string ceph::io_exerciser::IoSequence::get_name_with_seqseed() const
+{
+  return get_name() + " (seqseed " + std::to_string(get_seed()) + ")";
+}
+
 int IoSequence::get_step() const
 {
   return step;
@@ -133,6 +150,22 @@ std::unique_ptr<IoOp> IoSequence::increment_object_size()
   return BarrierOp::generate();
 }
 
+Sequence IoSequence::getNextSupportedSequenceId() const
+{
+  Sequence sequence = get_id();
+  ++sequence;
+  for (;sequence < Sequence::SEQUENCE_END;
+       ++sequence)
+  {
+    if (is_supported(sequence))
+    {
+      return sequence;
+    }
+  }
+
+  return Sequence::SEQUENCE_END;
+}
+
 std::unique_ptr<IoOp> IoSequence::next()
 {
   step++;
@@ -164,10 +197,15 @@ ceph::io_exerciser::Seq0::Seq0(std::pair<int,int> obj_size_range, int seed) :
   length = 1 + rng(obj_size - 1);
 }
 
+Sequence ceph::io_exerciser::Seq0::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ0;
+}
+
 std::string ceph::io_exerciser::Seq0::get_name() const
 {
   return "Sequential reads of length " + std::to_string(length) +
-    " with queue depth 1 (seqseed " + std::to_string(get_seed()) + ")";
+    " with queue depth 1";
 }
 
 std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq0::_next()
@@ -197,10 +235,14 @@ ceph::io_exerciser::Seq1::Seq1(std::pair<int,int> obj_size_range, int seed) :
   count = 3 * obj_size;
 }
 
+Sequence ceph::io_exerciser::Seq1::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ1;
+}
+
 std::string ceph::io_exerciser::Seq1::get_name() const
 {
-  return "Random offset, random length read/write I/O with queue depth 1 (seqseed "
-    + std::to_string(get_seed()) + ")";
+  return "Random offset, random length read/write I/O with queue depth 1";
 }
 
 std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq1::_next()
@@ -228,7 +270,15 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq1::_next()
 
 
 ceph::io_exerciser::Seq2::Seq2(std::pair<int,int> obj_size_range, int seed) :
-  IoSequence(obj_size_range, seed), offset(0), length(0) {}
+  IoSequence(obj_size_range, seed), offset(0), length(0)
+{
+
+}
+
+Sequence ceph::io_exerciser::Seq2::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ2;
+}
 
 std::string ceph::io_exerciser::Seq2::get_name() const
 {
@@ -258,6 +308,11 @@ ceph::io_exerciser::Seq3::Seq3(std::pair<int,int> obj_size_range, int seed) :
   set_min_object_size(2);
 }
 
+Sequence ceph::io_exerciser::Seq3::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ3;
+}
+
 std::string ceph::io_exerciser::Seq3::get_name() const
 {
   return "Permutations of offset 2-region 1-block read I/O";
@@ -286,6 +341,11 @@ ceph::io_exerciser::Seq4::Seq4(std::pair<int,int> obj_size_range, int seed) :
   set_min_object_size(3);
 }
 
+Sequence ceph::io_exerciser::Seq4::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ4;
+}
+
 std::string ceph::io_exerciser::Seq4::get_name() const
 {
   return "Permutations of offset 3-region 1-block read I/O";
@@ -310,7 +370,15 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq4::_next()
 
 ceph::io_exerciser::Seq5::Seq5(std::pair<int,int> obj_size_range, int seed) :
   IoSequence(obj_size_range, seed), offset(0), length(1),
-  doneread(false), donebarrier(false) {}
+  doneread(false), donebarrier(false)
+{
+
+}
+
+Sequence ceph::io_exerciser::Seq5::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ5;
+}
 
 std::string ceph::io_exerciser::Seq5::get_name() const
 {
@@ -348,7 +416,15 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq5::_next()
 
 ceph::io_exerciser::Seq6::Seq6(std::pair<int,int> obj_size_range, int seed) :
   IoSequence(obj_size_range, seed), offset(0), length(1),
-  doneread(false), donebarrier(false) {}
+  doneread(false), donebarrier(false)
+{
+
+}
+
+Sequence ceph::io_exerciser::Seq6::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ6;
+}
 
 std::string ceph::io_exerciser::Seq6::get_name() const
 {
@@ -394,6 +470,11 @@ ceph::io_exerciser::Seq7::Seq7(std::pair<int,int> obj_size_range, int seed) :
   offset = obj_size;
 }
 
+Sequence ceph::io_exerciser::Seq7::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ7;
+}
+
 std::string ceph::io_exerciser::Seq7::get_name() const
 {
   return "Permutations of offset 2-region 1-block writes";
@@ -433,6 +514,11 @@ ceph::io_exerciser::Seq8::Seq8(std::pair<int,int> obj_size_range, int seed) :
   set_min_object_size(3);
 }
 
+Sequence ceph::io_exerciser::Seq8::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ8;
+}
+
 std::string ceph::io_exerciser::Seq8::get_name() const
 {
   return "Permutations of offset 3-region 1-block write I/O";
@@ -472,6 +558,11 @@ ceph::io_exerciser::Seq9::Seq9(std::pair<int,int> obj_size_range, int seed) :
 
 }
 
+Sequence ceph::io_exerciser::Seq9::get_id() const
+{
+  return Sequence::SEQUENCE_SEQ9;
+}
+
 std::string ceph::io_exerciser::Seq9::get_name() const
 {
   return "Permutations of offset and length write I/O";

diff --git a/src/common/io_exerciser/IoSequence.h b/src/common/io_exerciser/IoSequence.h
index ccce4336c226fe7063cb2513e312adfaee348439..46116c5a1c6edbb2d3b2822a24f9302cdb14e34e 100644 (file)
--- a/src/common/io_exerciser/IoSequence.h
+++ b/src/common/io_exerciser/IoSequence.h
@@ -42,7 +42,8 @@ namespace ceph {
       SEQUENCE_SEQ7,
       SEQUENCE_SEQ8,
       SEQUENCE_SEQ9,
-      //
+      SEQUENCE_SEQ10,
+
       SEQUENCE_END,
       SEQUENCE_BEGIN = SEQUENCE_SEQ0
     };
@@ -60,12 +61,16 @@ namespace ceph {
     public:
       virtual ~IoSequence() = default;
 
+      virtual Sequence get_id() const = 0;
+      virtual std::string get_name_with_seqseed() const;
       virtual std::string get_name() const = 0;
       int get_step() const;
       int get_seed() const;
 
+      virtual Sequence getNextSupportedSequenceId() const;
       virtual std::unique_ptr<IoOp> next();
 
+      virtual bool is_supported(Sequence sequence) const;
       static std::unique_ptr<IoSequence>
         generate_sequence(Sequence s,
                           std::pair<int,int> obj_size_range,
@@ -98,6 +103,7 @@ namespace ceph {
     public:
       Seq0(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
 
@@ -110,8 +116,9 @@ namespace ceph {
     public:
       Seq1(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
-      std::unique_ptr<IoOp> _next();
+      std::unique_ptr<IoOp> _next() override;
 
     private:
       int count;
@@ -121,6 +128,7 @@ namespace ceph {
     public:
       Seq2(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
 
@@ -133,6 +141,7 @@ namespace ceph {
     public:
       Seq3(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
     private:
@@ -144,6 +153,7 @@ namespace ceph {
     public:
       Seq4(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
 
@@ -156,6 +166,7 @@ namespace ceph {
     public:
       Seq5(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
 
@@ -170,6 +181,7 @@ namespace ceph {
     public:
       Seq6(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
 
@@ -184,6 +196,7 @@ namespace ceph {
     public:
       Seq7(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
 
@@ -197,6 +210,7 @@ namespace ceph {
     public:
       Seq8(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
       std::unique_ptr<IoOp> _next() override;
     private:
@@ -216,8 +230,8 @@ namespace ceph {
     public:
       Seq9(std::pair<int,int> obj_size_range, int seed);
 
+      Sequence get_id() const override;
       std::string get_name() const override;
-
       std::unique_ptr<IoOp> _next() override;
     };
   }

diff --git a/src/common/io_exerciser/JsonStructures.cc b/src/common/io_exerciser/JsonStructures.cc
index 12d792a0cbbfa230415a39d874c7ef6b3a177acc..ba41c226264f4c1ec2f8ef63b59ea2001f668430 100644 (file)
--- a/src/common/io_exerciser/JsonStructures.cc
+++ b/src/common/io_exerciser/JsonStructures.cc
@@ -1,6 +1,7 @@
 #include "JsonStructures.h"
 
 #include "common/ceph_json.h"
+#include "OpType.h"
 
 using namespace ceph::io_exerciser::json;
 
@@ -93,9 +94,147 @@ void OSDMapReply::dump() const
   formatter->close_section();
 }
 
-OSDECProfileSetRequest::OSDECProfileSetRequest(const std::string& name,
-                                               std::vector<std::string> profile,
-                                               std::shared_ptr<ceph::Formatter> formatter) :
+ceph::io_exerciser::json::OSDPoolGetRequest
+  ::OSDPoolGetRequest(const std::string& pool_name,
+                      std::shared_ptr<ceph::Formatter> formatter) :
+  JSONStructure(formatter),
+  pool(pool_name)
+{
+
+}
+
+ceph::io_exerciser::json::OSDPoolGetRequest
+  ::OSDPoolGetRequest(JSONObj* obj,
+                      std::shared_ptr<ceph::Formatter> formatter) :
+  JSONStructure(formatter)
+{
+  ceph::io_exerciser::json::OSDPoolGetRequest::decode_json(obj);
+}
+
+void ceph::io_exerciser::json::OSDPoolGetRequest::decode_json(JSONObj* obj) {
+  JSONDecoder::decode_json("prefix", prefix, obj);
+  JSONDecoder::decode_json("pool", pool, obj);
+  JSONDecoder::decode_json("var", var, obj);
+  JSONDecoder::decode_json("format", format, obj);
+}
+
+void ceph::io_exerciser::json::OSDPoolGetRequest::dump() const
+{
+  formatter->open_object_section("OSDPoolGetRequest");
+  ::encode_json("prefix", prefix, formatter.get());
+  ::encode_json("pool", pool, formatter.get());
+  ::encode_json("var", var, formatter.get());
+  ::encode_json("format", format, formatter.get());
+  formatter->close_section();
+}
+
+ceph::io_exerciser::json::OSDPoolGetReply
+  ::OSDPoolGetReply(std::shared_ptr<ceph::Formatter> formatter) :
+  JSONStructure(formatter)
+{
+
+}
+
+void ceph::io_exerciser::json::OSDPoolGetReply::decode_json(JSONObj* obj) {
+  JSONDecoder::decode_json("erasure_code_profile", erasure_code_profile, obj);
+}
+
+void ceph::io_exerciser::json::OSDPoolGetReply::dump() const
+{
+  formatter->open_object_section("OSDPoolGetReply");
+  ::encode_json("erasure_code_profile", erasure_code_profile, formatter.get());
+  formatter->close_section();
+}
+
+ceph::io_exerciser::json::OSDECProfileGetRequest
+  ::OSDECProfileGetRequest(const std::string& profile_name,
+                           std::shared_ptr<ceph::Formatter> formatter) :
+  JSONStructure(formatter),
+  name(profile_name)
+{
+
+}
+
+ceph::io_exerciser::json::OSDECProfileGetRequest
+  ::OSDECProfileGetRequest(std::shared_ptr<ceph::Formatter> formatter) :
+  JSONStructure(formatter)
+{
+
+}
+
+void ceph::io_exerciser::json::OSDECProfileGetRequest::decode_json(JSONObj* obj) {
+  JSONDecoder::decode_json("prefix", prefix, obj);
+  JSONDecoder::decode_json("name", name, obj);
+  JSONDecoder::decode_json("format", format, obj);
+}
+
+void ceph::io_exerciser::json::OSDECProfileGetRequest::dump() const
+{
+  formatter->open_object_section("OSDECProfileGetRequest");
+  ::encode_json("prefix", prefix, formatter.get());
+  ::encode_json("name", name, formatter.get());
+  ::encode_json("format", format, formatter.get());
+  formatter->close_section();
+}
+
+ceph::io_exerciser::json::OSDECProfileGetReply
+  ::OSDECProfileGetReply(std::shared_ptr<ceph::Formatter> formatter) :
+  JSONStructure(formatter)
+{
+
+}
+
+void ceph::io_exerciser::json::OSDECProfileGetReply::decode_json(JSONObj* obj) {
+  JSONDecoder::decode_json("crush-device-class", crush_device_class, obj);
+  JSONDecoder::decode_json("crush-failure-domain", crush_failure_domain, obj);
+  JSONDecoder::decode_json("crush-num-failure-domains",
+                           crush_num_failure_domains,
+                           obj);
+  JSONDecoder::decode_json("crush-osds-per-failure-domain",
+                           crush_osds_per_failure_domain,
+                           obj);
+  JSONDecoder::decode_json("crush-root", crush_root, obj);
+  JSONDecoder::decode_json("jerasure-per-chunk-alignment",
+                           jerasure_per_chunk_alignment,
+                           obj);
+  JSONDecoder::decode_json("k", k, obj);
+  JSONDecoder::decode_json("m", m, obj);
+  JSONDecoder::decode_json("plugin", plugin, obj);
+  JSONDecoder::decode_json("technique", technique, obj);
+  JSONDecoder::decode_json("w", w, obj);
+}
+
+void ceph::io_exerciser::json::OSDECProfileGetReply::dump() const
+{
+  formatter->open_object_section("OSDECProfileGetReply");
+  ::encode_json("crush-device-class",
+                crush_device_class,
+                formatter.get());
+  ::encode_json("crush-failure-domain",
+                crush_failure_domain,
+                formatter.get());
+  ::encode_json("crush-num-failure-domains",
+                crush_num_failure_domains,
+                formatter.get());
+  ::encode_json("crush-osds-per-failure-domain",
+                crush_osds_per_failure_domain,
+                formatter.get());
+  ::encode_json("crush-root", crush_root, formatter.get());
+  ::encode_json("jerasure-per-chunk-alignment",
+                jerasure_per_chunk_alignment,
+                formatter.get());
+  ::encode_json("k", k, formatter.get());
+  ::encode_json("m", m, formatter.get());
+  ::encode_json("plugin", plugin, formatter.get());
+  ::encode_json("technique", technique, formatter.get());
+  ::encode_json("w", w, formatter.get());
+  formatter->close_section();
+}
+
+ceph::io_exerciser::json::OSDECProfileSetRequest
+  ::OSDECProfileSetRequest(const std::string& name,
+                           const std::vector<std::string>& profile,
+                           std::shared_ptr<ceph::Formatter> formatter) :
   JSONStructure(formatter),
   name(name),
   profile(profile)
@@ -140,7 +279,6 @@ OSDECPoolCreateRequest
   ::OSDECPoolCreateRequest(std::shared_ptr<ceph::Formatter> formatter) :
   JSONStructure(formatter)
 {
-
 }
 
 void OSDECPoolCreateRequest::decode_json(JSONObj* obj)
@@ -178,7 +316,6 @@ OSDSetRequest::OSDSetRequest(const std::string& key,
 OSDSetRequest::OSDSetRequest(std::shared_ptr<ceph::Formatter> formatter) :
   JSONStructure(formatter)
 {
-
 }
 
 void OSDSetRequest::decode_json(JSONObj* obj)
@@ -203,7 +340,7 @@ BalancerOffRequest::BalancerOffRequest(std::shared_ptr<ceph::Formatter> formatte
 
 }
 
-void BalancerOffRequest::decode_json(JSONObj* obj)
+void ceph::io_exerciser::json::BalancerOffRequest::decode_json(JSONObj* obj)
 {
   JSONDecoder::decode_json("prefix", prefix, obj);
 }
@@ -222,7 +359,7 @@ BalancerStatusRequest
 
 }
 
-void BalancerStatusRequest::decode_json(JSONObj* obj)
+void ceph::io_exerciser::json::BalancerStatusRequest::decode_json(JSONObj* obj)
 {
   JSONDecoder::decode_json("prefix", prefix, obj);
 }
@@ -332,12 +469,18 @@ InjectECErrorRequest::InjectECErrorRequest(InjectOpType injectOpType,
 {
   switch(injectOpType)
   {
-    case InjectOpType::Read:
+    case InjectOpType::ReadEIO:
+      [[ fallthrough ]];
+    case InjectOpType::ReadMissingShard:
       prefix = "injectecreaderr";
       break;
-    case InjectOpType::Write:
+    case InjectOpType::WriteFailAndRollback:
+      [[ fallthrough ]];
+    case InjectOpType::WriteOSDAbort:
       prefix = "injectecwriteerr";
       break;
+    default:
+      ceph_abort_msg("Invalid OP type to inject");
   }
 }
 
@@ -388,12 +531,18 @@ InjectECClearErrorRequest::InjectECClearErrorRequest(InjectOpType injectOpType,
 {
   switch(injectOpType)
   {
-    case InjectOpType::Read:
+        case InjectOpType::ReadEIO:
+      [[ fallthrough ]];
+    case InjectOpType::ReadMissingShard:
       prefix = "injectecclearreaderr";
       break;
-    case InjectOpType::Write:
+    case InjectOpType::WriteFailAndRollback:
+      [[ fallthrough ]];
+    case InjectOpType::WriteOSDAbort:
       prefix = "injectecclearwriteerr";
       break;
+    default:
+      ceph_abort_msg("Invalid OP type to inject");
   }
 }
 

diff --git a/src/common/io_exerciser/JsonStructures.h b/src/common/io_exerciser/JsonStructures.h
index 7e1b295919d7a09a3d76de77c572033e3fefbdc9..b439845b3ab2a1ed4ea9d7a71af399cce43a8482 100644 (file)
--- a/src/common/io_exerciser/JsonStructures.h
+++ b/src/common/io_exerciser/JsonStructures.h
@@ -4,6 +4,8 @@
 
 #include "include/types.h"
 
+#include "OpType.h"
+
 /* Overview
  *
  * class JSONStructure
@@ -82,11 +84,72 @@ namespace ceph
           void dump() const;
       };
 
+      class OSDPoolGetRequest : public JSONStructure
+      {
+        public:
+          OSDPoolGetRequest(const std::string& pool_name, std::shared_ptr<ceph::Formatter> formatter = std::make_shared<JSONFormatter>(false));
+          OSDPoolGetRequest(JSONObj* obj, std::shared_ptr<ceph::Formatter> formatter = std::make_shared<JSONFormatter>(false));
+
+          std::string prefix = "osd pool get";
+          std::string pool;
+          std::string var = "erasure_code_profile";
+          std::string format = "json";
+
+          void decode_json(JSONObj* obj) override;
+          void dump() const override;
+      };
+
+      class OSDPoolGetReply : public JSONStructure
+      {
+        public:
+          OSDPoolGetReply(std::shared_ptr<ceph::Formatter> formatter = std::make_shared<JSONFormatter>(false));
+
+          std::string erasure_code_profile;
+
+          void decode_json(JSONObj *obj);
+          void dump() const;
+      };
+
+      class OSDECProfileGetRequest : public JSONStructure
+      {
+        public:
+          OSDECProfileGetRequest(const std::string& profile_name, std::shared_ptr<ceph::Formatter> formatter = std::make_shared<JSONFormatter>(false));
+          OSDECProfileGetRequest(std::shared_ptr<ceph::Formatter> formatter = std::make_shared<JSONFormatter>(false));
+
+          std::string prefix = "osd pool get";
+          std::string name;
+          std::string format = "json";
+
+          void decode_json(JSONObj* obj) override;
+          void dump() const override;
+      };
+
+      class OSDECProfileGetReply : public JSONStructure
+      {
+        public:
+          OSDECProfileGetReply(std::shared_ptr<ceph::Formatter> formatter = std::make_shared<JSONFormatter>(false));
+
+          std::string crush_device_class;
+          std::string crush_failure_domain;
+          int crush_num_failure_domains;
+          int crush_osds_per_failure_domain;
+          std::string crush_root;
+          bool jerasure_per_chunk_alignment;
+          int k;
+          int m;
+          std::string plugin;
+          std::string technique;
+          std::string w;
+
+          void decode_json(JSONObj *obj);
+          void dump() const;
+      };
+
       class OSDECProfileSetRequest : public JSONStructure
       {
         public:
           OSDECProfileSetRequest(const std::string& name,
-                                 std::vector<std::string> profile,
+                                 const std::vector<std::string>& profile,
                                  std::shared_ptr<Formatter> formatter
                                     = std::make_shared<JSONFormatter>(false));
           OSDECProfileSetRequest(std::shared_ptr<Formatter> formatter
@@ -203,12 +266,6 @@ namespace ceph
           void dump() const override;
       };
 
-      enum class InjectOpType
-      {
-        Read,
-        Write
-      };
-
       class InjectECErrorRequest : public JSONStructure
       {
         public:

diff --git a/src/common/io_exerciser/OpType.h b/src/common/io_exerciser/OpType.h
index 737973e0455565d4858f916720fc162efd1a5ea4..84901b372c2350090c5b52cbfcb791d36e909755 100644 (file)
--- a/src/common/io_exerciser/OpType.h
+++ b/src/common/io_exerciser/OpType.h
@@ -35,6 +35,14 @@ namespace ceph
       ClearReadErrorInject,   // Op to tell OSD to clear read error injects
       ClearWriteErrorInject   // Op to tell OSD to clear write error injects
     };
+
+    enum class InjectOpType {
+      None,
+      ReadEIO,
+      ReadMissingShard,
+      WriteFailAndRollback,
+      WriteOSDAbort
+    };
   }
 }
 

diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc
index be91b3c27cad957e74ecf0f91207dba7a107dd39..03cb2d26d3f0c9f0bd78e8bfffc7e8aefe097e22 100644 (file)
--- a/src/common/io_exerciser/RadosIo.cc
+++ b/src/common/io_exerciser/RadosIo.cc
@@ -340,6 +340,7 @@ void RadosIo::applyInjectOp(IoOp& op)
   auto formatter = std::make_shared<JSONFormatter>(false);
 
   int osd = -1;
+  std::vector<int> shard_order;
 
   ceph::io_exerciser::json::OSDMapRequest osdMapRequest(pool,
                                                         get_oid(),
@@ -359,6 +360,9 @@ void RadosIo::applyInjectOp(IoOp& op)
   reply.decode_json(&p);
 
   osd = reply.acting_primary;
+  shard_order = reply.acting;
+
+  InjectOpType injectOpType;
 
   switch(op.getOpType())
   {
@@ -366,7 +370,20 @@ void RadosIo::applyInjectOp(IoOp& op)
     {
       InjectReadErrorOp& errorOp = static_cast<InjectReadErrorOp&>(op);
 
-      ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(json::InjectOpType::Read,
+      if (errorOp.type == 0)
+      {
+        injectOpType = InjectOpType::ReadEIO;
+      }
+      else if (errorOp.type == 1)
+      {
+        injectOpType = InjectOpType::ReadMissingShard;
+      }
+      else
+      {
+        ceph_abort_msg("Unsupported inject type");
+      }
+
+      ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(injectOpType,
                                                                         pool,
                                                                         oid,
                                                                         errorOp.shard,
@@ -383,7 +400,23 @@ void RadosIo::applyInjectOp(IoOp& op)
     {
       InjectWriteErrorOp& errorOp = static_cast<InjectWriteErrorOp&>(op);
 
-      ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(json::InjectOpType::Write,
+      if (errorOp.type == 0)
+      {
+        injectOpType = InjectOpType::WriteFailAndRollback;
+      }
+      else if (errorOp.type == 3)
+      {
+        injectOpType = InjectOpType::WriteOSDAbort;
+
+         // This inject is sent directly to the shard we want to inject the error on
+        osd = shard_order[errorOp.shard];
+      }
+      else
+      {
+        ceph_abort("Unsupported inject type");
+      }
+
+      ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(injectOpType,
                                                                         pool,
                                                                         oid,
                                                                         errorOp.shard,
@@ -400,7 +433,20 @@ void RadosIo::applyInjectOp(IoOp& op)
     {
       ClearReadErrorInjectOp& errorOp = static_cast<ClearReadErrorInjectOp&>(op);
 
-      ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(json::InjectOpType::Read,
+      if (errorOp.type == 0)
+      {
+        injectOpType = InjectOpType::ReadEIO;
+      }
+      else if (errorOp.type == 1)
+      {
+        injectOpType = InjectOpType::ReadMissingShard;
+      }
+      else
+      {
+        ceph_abort("Unsupported inject type");
+      }
+
+      ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(injectOpType,
                                                                            pool,
                                                                            oid,
                                                                            errorOp.shard,
@@ -414,7 +460,20 @@ void RadosIo::applyInjectOp(IoOp& op)
     {
       ClearReadErrorInjectOp& errorOp = static_cast<ClearReadErrorInjectOp&>(op);
 
-      ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(json::InjectOpType::Write,
+      if (errorOp.type == 0)
+      {
+        injectOpType = InjectOpType::WriteFailAndRollback;
+      }
+      else if (errorOp.type == 3)
+      {
+        injectOpType = InjectOpType::WriteOSDAbort;
+      }
+      else
+      {
+        ceph_abort("Unsupported inject type");
+      }
+
+      ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(injectOpType,
                                                                            pool,
                                                                            oid,
                                                                            errorOp.shard,

diff --git a/src/test/osd/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence.cc
index 0a45739a43db97b7f4da9d0c442386eeb8889ae4..18852b6aa3168d128b4f40e721498f18df2da258 100644 (file)
--- a/src/test/osd/ceph_test_rados_io_sequence.cc
+++ b/src/test/osd/ceph_test_rados_io_sequence.cc
@@ -26,6 +26,7 @@
 #include "common/io_exerciser/RadosIo.h"
 #include "common/io_exerciser/IoOp.h"
 #include "common/io_exerciser/IoSequence.h"
+#include "common/io_exerciser/EcIoSequence.h"
 #include "common/io_exerciser/JsonStructures.h"
 
 #include "json_spirit/json_spirit.h"
@@ -186,6 +187,8 @@ namespace {
         "number of threads of I/O per object (default 1)")
       ("parallel,p", po::value<int>()->default_value(1),
         "number of objects to exercise in parallel")
+      ("testrecovery",
+        "Inject errors during sequences to test recovery processes of OSDs")
       ("interactive",
         "interactive mode, execute IO commands from stdin")
       ("allow_pool_autoscaling",
@@ -330,7 +333,9 @@ const std::pair<ceph::io_exerciser::Sequence,ceph::io_exerciser::Sequence>
   if (force_value.has_value())
   {
     return *force_value;
-  } else {
+  }
+  else
+  {
     return std::make_pair(ceph::io_exerciser::Sequence::SEQUENCE_BEGIN,
                           ceph::io_exerciser::Sequence::SEQUENCE_END);
   }
@@ -396,12 +401,38 @@ const std::string ceph::io_sequence::tester::SelectECPool::choose()
 {
   std::pair<int,int> value;
   if (!skm.isForced() && force_value.has_value()) {
+    int rc;
+    bufferlist inbl, outbl;
+    auto formatter = std::make_shared<JSONFormatter>(false);
+
+    ceph::io_exerciser::json::OSDPoolGetRequest osdPoolGetRequest(*force_value, formatter);
+    rc = rados.mon_command(osdPoolGetRequest.encode_json(), inbl, &outbl, nullptr);
+    ceph_assert(rc == 0);
+
+    JSONParser p;
+    bool success = p.parse(outbl.c_str(), outbl.length());
+    ceph_assert(success);
+
+    ceph::io_exerciser::json::OSDPoolGetReply osdPoolGetReply(formatter);
+    osdPoolGetReply.decode_json(&p);
+
+    ceph::io_exerciser::json::OSDECProfileGetRequest osdECProfileGetRequest(osdPoolGetReply.erasure_code_profile, formatter);
+    rc = rados.mon_command(osdECProfileGetRequest.encode_json(), inbl, &outbl, nullptr);
+    ceph_assert(rc == 0);
+
+    success = p.parse(outbl.c_str(), outbl.length());
+    ceph_assert(success);
+
+    ceph::io_exerciser::json::OSDECProfileGetReply reply(formatter);
+    reply.decode_json(&p);
+    k = reply.k;
+    m = reply.m;
     return *force_value;
   } else {
     value = skm.choose();
   }
-  int k = value.first;
-  int m = value.second;
+  k = value.first;
+  m = value.second;
 
   const std::string plugin = std::string(spl.choose());
   const uint64_t chunk_size = scs.choose();
@@ -529,16 +560,19 @@ ceph::io_sequence::tester::TestObject::TestObject( const std::string oid,
                         ceph::condition_variable& cond,
                         bool dryrun,
                         bool verbose,
-                        std::optional<int>  seqseed) :
-  rng(rng), verbose(verbose), seqseed(seqseed)
+                        std::optional<int>  seqseed,
+                        bool testrecovery) :
+  rng(rng), verbose(verbose), seqseed(seqseed), testrecovery(testrecovery)
 {
   if (dryrun) {
-    verbose = true;
     exerciser_model = std::make_unique<ceph::io_exerciser::ObjectModel>(oid,
                                                                         sbs.choose(),
                                                                         rng());
   } else {
     const std::string pool = spo.choose();
+    poolK = spo.getChosenK();
+    poolM = spo.getChosenM();
+
     int threads = snt.choose();
 
     bufferlist inbl, outbl;
@@ -581,14 +615,27 @@ ceph::io_sequence::tester::TestObject::TestObject( const std::string oid,
   obj_size_range = sos.choose();
   seq_range = ssr.choose();
   curseq = seq_range.first;
-  seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq,
-                                                          obj_size_range,
-                                                          seqseed.value_or(rng()));
+
+  if (testrecovery)
+  {
+    seq = ceph::io_exerciser::EcIoSequence::generate_sequence(curseq,
+                                                              obj_size_range,
+                                                              poolK,
+                                                              poolM,
+                                                              seqseed.value_or(rng()));
+  }
+  else
+  {
+    seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq,
+                                                            obj_size_range,
+                                                            seqseed.value_or(rng()));
+  }
+
   op = seq->next();
   done = false;
   dout(0) << "== " << exerciser_model->get_oid() << " "
           << curseq << " "
-          << seq->get_name()
+          << seq->get_name_with_seqseed()
           << " ==" <<dendl;
 }
 
@@ -616,8 +663,8 @@ bool ceph::io_sequence::tester::TestObject::next()
     exerciser_model->applyIoOp(*op);
     if (op->getOpType() == ceph::io_exerciser::OpType::Done)
     {
-      ++curseq;
-      if (curseq == seq_range.second)
+      curseq = seq->getNextSupportedSequenceId();
+      if (curseq >= seq_range.second)
       {
         done = true;
         dout(0) << exerciser_model->get_oid()
@@ -626,11 +673,22 @@ bool ceph::io_sequence::tester::TestObject::next()
       }
       else
       {
-        seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq,
-                                                                obj_size_range,
-                                                                seqseed.value_or(rng()));
+        if (testrecovery)
+        {
+          seq = ceph::io_exerciser::EcIoSequence::generate_sequence(curseq,
+                                                                    obj_size_range,
+                                                                    poolK, poolM,
+                                                                    seqseed.value_or(rng()));
+        }
+        else
+        {
+          seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq,
+                                                                  obj_size_range,
+                                                                  seqseed.value_or(rng()));
+        }
+
         dout(0) << "== " << exerciser_model->get_oid() << " "
-                << curseq << " " << seq->get_name()
+                << curseq << " " << seq->get_name_with_seqseed()
                 << " ==" <<dendl;
         op = seq->next();
       }
@@ -681,6 +739,7 @@ ceph::io_sequence::tester::TestRunner::TestRunner(po::variables_map& vm,
   num_objects = vm["parallel"].as<int>();
   object_name = vm["object"].as<std::string>();
   interactive = vm.contains("interactive");
+  testrecovery = vm.contains("testrecovery");
 
   allow_pool_autoscaling = vm.contains("allow_pool_autoscaling");
   allow_pool_balancer = vm.contains("allow_pool_balancer");
@@ -716,20 +775,30 @@ void ceph::io_sequence::tester::TestRunner::help()
   }
 }
 
-void ceph::io_sequence::tester::TestRunner::list_sequence()
+void ceph::io_sequence::tester::TestRunner::list_sequence(bool testrecovery)
 {
   // List seqeunces
   std::pair<int,int> obj_size_range = sos.choose();
-  for (ceph::io_exerciser::Sequence s
-        = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN;
-        s < ceph::io_exerciser::Sequence::SEQUENCE_END; ++s)
+  ceph::io_exerciser::Sequence s = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN;
+  std::unique_ptr<ceph::io_exerciser::IoSequence> seq;
+  if (testrecovery)
   {
-    std::unique_ptr<ceph::io_exerciser::IoSequence> seq =
-    ceph::io_exerciser::IoSequence::generate_sequence(s,
-                                                      obj_size_range,
-                                                      seqseed.value_or(rng()));
-    dout(0) << s << " " << seq->get_name() << dendl;
+    seq = ceph::io_exerciser::EcIoSequence::generate_sequence(s, obj_size_range,
+                                                              spo.getChosenK(),
+                                                              spo.getChosenM(),
+                                                              seqseed.value_or(rng()));
   }
+  else
+  {
+    seq = ceph::io_exerciser::IoSequence::generate_sequence(s, obj_size_range,
+                                                            seqseed.value_or(rng()));
+  }
+
+  do
+  {
+    dout(0) << s << " " << seq->get_name_with_seqseed() << dendl;
+    s = seq->getNextSupportedSequenceId();
+  } while (s != ceph::io_exerciser::Sequence::SEQUENCE_END);
 }
 
 void ceph::io_sequence::tester::TestRunner::clear_tokens()
@@ -801,7 +870,7 @@ bool ceph::io_sequence::tester::TestRunner::run_test()
   }
   else if (show_sequence)
   {
-    list_sequence();
+    list_sequence(testrecovery);
     return true;
   }
   else if (interactive)
@@ -1041,7 +1110,7 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test()
             sbs, spo, sos, snt, ssr,
             rng, lock, cond,
             dryrun, verbose,
-            seqseed
+            seqseed, testrecovery
       )
     );
   }

diff --git a/src/test/osd/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence.h
index e34fdf32d0b60e4ceffd31c41b482bd0c2f6995e..265fa3e23a46f10cffc8f2f72ae3a174214721bf 100644 (file)
--- a/src/test/osd/ceph_test_rados_io_sequence.h
+++ b/src/test/osd/ceph_test_rados_io_sequence.h
@@ -251,6 +251,8 @@ namespace ceph
       bool get_allow_pool_balancer() { return allow_pool_balancer; }
       bool get_allow_pool_deep_scrubbing() { return allow_pool_deep_scrubbing; }
       bool get_allow_pool_scrubbing() { return allow_pool_scrubbing; }
+      int getChosenK() const { return k; }
+      int getChosenM() const { return m; }
 
     private:
       void create_pool(librados::Rados& rados,
@@ -266,7 +268,9 @@ namespace ceph
       bool allow_pool_balancer;
       bool allow_pool_deep_scrubbing;
       bool allow_pool_scrubbing;
-      
+      int k;
+      int m;
+
       SelectErasureKM skm;
       SelectErasurePlugin spl;
       SelectErasureChunkSize scs;
@@ -288,7 +292,8 @@ namespace ceph
                   ceph::condition_variable& cond,
                   bool dryrun,
                   bool verbose,
-                  std::optional<int>  seqseed);
+                  std::optional<int>  seqseed,
+                  bool testRecovery);
       
       int get_num_io();
       bool readyForIo();
@@ -307,6 +312,9 @@ namespace ceph
       ceph::util::random_number_generator<int>& rng;
       bool verbose;
       std::optional<int> seqseed;
+      int poolK;
+      int poolM;
+      bool testrecovery;
     };
 
     class TestRunner
@@ -342,6 +350,8 @@ namespace ceph
       std::optional<int> seqseed;
       bool interactive;
 
+      bool testrecovery;
+
       bool allow_pool_autoscaling;
       bool allow_pool_balancer;
       bool allow_pool_deep_scrubbing;
@@ -368,7 +378,7 @@ namespace ceph
       bool run_interactive_test();
 
       void help();
-      void list_sequence();
+      void list_sequence(bool testrecovery);
     };
   }
 }