Add a new flag to the IO exerciser which enables live consistency

checking during IO sequences (disabled by default).

Signed-off-by: Connor Fawcett <connorfa@uk.ibm.com>

diff --git a/src/common/io_exerciser/EcIoSequence.cc b/src/common/io_exerciser/EcIoSequence.cc
index 1878f2f7002f87de922744388f47168a725f02ef..2c61a7b381c154b3750bc3e1357d32ad7184b0a9 100644 (file)
--- a/src/common/io_exerciser/EcIoSequence.cc
+++ b/src/common/io_exerciser/EcIoSequence.cc
@@ -15,7 +15,8 @@ std::unique_ptr<IoSequence> EcIoSequence::generate_sequence(
     Sequence sequence, std::pair<int, int> obj_size_range,
     std::optional<std::pair<int, int>> km,
     std::optional<std::pair<std::string_view, std::string_view>> mappinglayers,
-    int seed) {
+    int seed,
+    bool check_consistency) {
   switch (sequence) {
     case Sequence::SEQUENCE_SEQ0:
       [[fallthrough]];
@@ -45,16 +46,16 @@ std::unique_ptr<IoSequence> EcIoSequence::generate_sequence(
       [[fallthrough]];
     case Sequence::SEQUENCE_SEQ14:
       return std::make_unique<ReadInjectSequence>(obj_size_range, seed,
-                                                  sequence, km, mappinglayers);
+                                                  sequence, km, mappinglayers, check_consistency);
     case Sequence::SEQUENCE_SEQ10:
-      return std::make_unique<Seq10>(obj_size_range, seed, km, mappinglayers);
+      return std::make_unique<Seq10>(obj_size_range, seed, km, mappinglayers, check_consistency);
     default:
       ceph_abort_msg("Unrecognised sequence");
   }
 }
 
-EcIoSequence::EcIoSequence(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed),
+EcIoSequence::EcIoSequence(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency),
       setup_inject(false),
       clear_inject(false),
       shard_to_inject(std::nullopt) {}
@@ -172,9 +173,10 @@ ceph::io_exerciser::ReadInjectSequence::ReadInjectSequence(
     int seed,
     Sequence s,
     std::optional<std::pair<int, int>> km,
-    std::optional<std::pair<std::string_view, std::string_view>> mappinglayers)
-    : EcIoSequence(obj_size_range, seed) {
-  child_sequence = IoSequence::generate_sequence(s, obj_size_range, seed);
+    std::optional<std::pair<std::string_view, std::string_view>> mappinglayers,
+    bool check_consistency)
+    : EcIoSequence(obj_size_range, seed, check_consistency) {
+  child_sequence = IoSequence::generate_sequence(s, obj_size_range, seed, check_consistency);
   select_random_data_shard_to_inject_read_error(km, mappinglayers);
   generate_random_read_inject_type();
 }
@@ -267,8 +269,9 @@ ceph::io_exerciser::ReadInjectSequence::_next() {
 ceph::io_exerciser::Seq10::Seq10(
     std::pair<int, int> obj_size_range, int seed,
     std::optional<std::pair<int, int>> km,
-    std::optional<std::pair<std::string_view, std::string_view>> mappinglayers)
-    : EcIoSequence(obj_size_range, seed),
+    std::optional<std::pair<std::string_view, std::string_view>> mappinglayers,
+    bool check_consistency)
+    : EcIoSequence(obj_size_range, seed, check_consistency),
       offset(0),
       length(1),
       inject_error_done(false),

diff --git a/src/common/io_exerciser/EcIoSequence.h b/src/common/io_exerciser/EcIoSequence.h
index 8082b2267a78eebf1e7a1d6f87b58b8b7a3ff6da..90f79da7315ec0cb678910c2c6c0695dd6a165b3 100644 (file)
--- a/src/common/io_exerciser/EcIoSequence.h
+++ b/src/common/io_exerciser/EcIoSequence.h
@@ -12,7 +12,8 @@ class EcIoSequence : public IoSequence {
       std::optional<std::pair<int, int>> km,
       std::optional<std::pair<std::string_view, std::string_view>>
           mappinglayers,
-      int seed);
+      int seed,
+      bool check_consistency);
 
  protected:
   bool setup_inject;
@@ -20,7 +21,7 @@ class EcIoSequence : public IoSequence {
   std::optional<uint64_t> shard_to_inject;
   InjectOpType inject_op_type;
 
-  EcIoSequence(std::pair<int, int> obj_size_range, int seed);
+  EcIoSequence(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   // Writes cannot be sent to injected on shard zero, so selections seperated
   // out
@@ -50,7 +51,8 @@ class ReadInjectSequence : public EcIoSequence {
       std::pair<int, int> obj_size_range, int seed, Sequence s,
       std::optional<std::pair<int, int>> km,
       std::optional<std::pair<std::string_view, std::string_view>>
-          mappinglayers);
+          mappinglayers,
+      bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -67,7 +69,8 @@ class Seq10 : public EcIoSequence {
   Seq10(std::pair<int, int> obj_size_range, int seed,
         std::optional<std::pair<int, int>> km,
         std::optional<std::pair<std::string_view, std::string_view>>
-            mappinglayers);
+            mappinglayers,
+        bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;

diff --git a/src/common/io_exerciser/IoOp.cc b/src/common/io_exerciser/IoOp.cc
index 1046d6aaa914a2af904f00b8ffafde0ebae93395..4b9fc57ce39879c78dfaf395df31d7e77c919123 100644 (file)
--- a/src/common/io_exerciser/IoOp.cc
+++ b/src/common/io_exerciser/IoOp.cc
@@ -10,6 +10,7 @@ using DoneOp = ceph::io_exerciser::DoneOp;
 using BarrierOp = ceph::io_exerciser::BarrierOp;
 using CreateOp = ceph::io_exerciser::CreateOp;
 using RemoveOp = ceph::io_exerciser::RemoveOp;
+using ConsistencyOp = ceph::io_exerciser::ConsistencyOp;
 using SingleReadOp = ceph::io_exerciser::SingleReadOp;
 using DoubleReadOp = ceph::io_exerciser::DoubleReadOp;
 using TripleReadOp = ceph::io_exerciser::TripleReadOp;
@@ -98,6 +99,16 @@ ceph::io_exerciser::ReadWriteOp<opType, numIOs>::ReadWriteOp(
   }
 }
 
+ConsistencyOp::ConsistencyOp() : TestOp<OpType::Consistency>() {}
+
+std::unique_ptr<ConsistencyOp> ConsistencyOp::generate() {
+  return std::make_unique<ConsistencyOp>();
+}
+
+std::string ConsistencyOp::to_string(uint64_t block_size) const {
+  return "Consistency";
+}
+
 template <OpType opType, int numIOs>
 std::string ceph::io_exerciser::ReadWriteOp<opType, numIOs>::to_string(
     uint64_t block_size) const {

diff --git a/src/common/io_exerciser/IoOp.h b/src/common/io_exerciser/IoOp.h
index 0cfc6c023092667f02c09814185b68fb3717c48b..ae58f6c40cd5099d79271bb992b3fa0a4469c2ae 100644 (file)
--- a/src/common/io_exerciser/IoOp.h
+++ b/src/common/io_exerciser/IoOp.h
@@ -61,6 +61,13 @@ class RemoveOp : public TestOp<OpType::Remove> {
   std::string to_string(uint64_t block_size) const override;
 };
 
+class ConsistencyOp : public TestOp<OpType::Consistency> {
+  public:
+   ConsistencyOp();
+   static std::unique_ptr<ConsistencyOp> generate();
+   std::string to_string(uint64_t block_size) const override;
+ };
+
 template <OpType opType, int numIOs>
 class ReadWriteOp : public TestOp<opType> {
  public:

diff --git a/src/common/io_exerciser/IoSequence.cc b/src/common/io_exerciser/IoSequence.cc
index 97289238fe6b78f58aaf36d010491bf4f2094d9c..2d4b80aa32c9b877f831c311ee73d30ed883d9de 100644 (file)
--- a/src/common/io_exerciser/IoSequence.cc
+++ b/src/common/io_exerciser/IoSequence.cc
@@ -2,6 +2,7 @@
 #include <algorithm>
 
 using IoOp = ceph::io_exerciser::IoOp;
+using OpType = ceph::io_exerciser::OpType;
 using Sequence = ceph::io_exerciser::Sequence;
 using IoSequence = ceph::io_exerciser::IoSequence;
 
@@ -65,54 +66,58 @@ bool IoSequence::is_supported(Sequence sequence) const {
 }
 
 std::unique_ptr<IoSequence> IoSequence::generate_sequence(
-    Sequence s, std::pair<int, int> obj_size_range, int seed) {
+    Sequence s, std::pair<int, int> obj_size_range, int seed, bool check_consistency) {
   switch (s) {
     case Sequence::SEQUENCE_SEQ0:
-      return std::make_unique<Seq0>(obj_size_range, seed);
+      return std::make_unique<Seq0>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ1:
-      return std::make_unique<Seq1>(obj_size_range, seed);
+      return std::make_unique<Seq1>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ2:
-      return std::make_unique<Seq2>(obj_size_range, seed);
+      return std::make_unique<Seq2>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ3:
-      return std::make_unique<Seq3>(obj_size_range, seed);
+      return std::make_unique<Seq3>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ4:
-      return std::make_unique<Seq4>(obj_size_range, seed);
+      return std::make_unique<Seq4>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ5:
-      return std::make_unique<Seq5>(obj_size_range, seed);
+      return std::make_unique<Seq5>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ6:
-      return std::make_unique<Seq6>(obj_size_range, seed);
+      return std::make_unique<Seq6>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ7:
-      return std::make_unique<Seq7>(obj_size_range, seed);
+      return std::make_unique<Seq7>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ8:
-      return std::make_unique<Seq8>(obj_size_range, seed);
+      return std::make_unique<Seq8>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ9:
-      return std::make_unique<Seq9>(obj_size_range, seed);
+      return std::make_unique<Seq9>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ10:
       ceph_abort_msg(
           "Sequence 10 only supported for erasure coded pools "
           "through the EcIoSequence interface");
       return nullptr;
     case Sequence::SEQUENCE_SEQ11:
-      return std::make_unique<Seq11>(obj_size_range, seed);
+      return std::make_unique<Seq11>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ12:
-      return std::make_unique<Seq12>(obj_size_range, seed);
+      return std::make_unique<Seq12>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ13:
-      return std::make_unique<Seq13>(obj_size_range, seed);
+      return std::make_unique<Seq13>(obj_size_range, seed, check_consistency);
     case Sequence::SEQUENCE_SEQ14:
-      return std::make_unique<Seq14>(obj_size_range, seed);
+      return std::make_unique<Seq14>(obj_size_range, seed, check_consistency);
     default:
       break;
   }
   return nullptr;
 }
 
-IoSequence::IoSequence(std::pair<int, int> obj_size_range, int seed)
+IoSequence::IoSequence(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
     : min_obj_size(obj_size_range.first),
       max_obj_size(obj_size_range.second),
       create(true),
       barrier(false),
       done(false),
       remove(false),
+      consistency(false),
+      consistency_in_progress(false),
+      consistency_request_sent(false),
+      check_consistency(check_consistency),
       obj_size(min_obj_size),
       step(-1),
       seed(seed) {
@@ -161,6 +166,26 @@ std::unique_ptr<IoOp> IoSequence::increment_object_size() {
   return BarrierOp::generate();
 }
 
+std::unique_ptr<IoOp> IoSequence::process_remove() {
+  if (check_consistency) {
+    if (!consistency_in_progress) {
+      consistency_in_progress = true;
+      consistency_request_sent = true;
+      return ConsistencyOp::generate();
+    }
+
+    if (consistency_request_sent) {
+      consistency_request_sent = false;
+      return BarrierOp::generate();
+    }
+  }
+
+  // Getting here means consistency and barriers have been applied if required
+  remove = false;
+  consistency_in_progress = false;
+  return RemoveOp::generate();
+}
+
 Sequence IoSequence::getNextSupportedSequenceId() const {
   Sequence sequence = get_id();
   ++sequence;
@@ -172,17 +197,20 @@ Sequence IoSequence::getNextSupportedSequenceId() const {
 
   return Sequence::SEQUENCE_END;
 }
-
 std::unique_ptr<IoOp> IoSequence::next() {
   step++;
   if (remove) {
-    remove = false;
-    return RemoveOp::generate();
+    return process_remove();
   }
   if (barrier) {
     barrier = false;
     return BarrierOp::generate();
   }
+  if (consistency) {
+    consistency = false;
+    barrier = true;
+    return ConsistencyOp::generate();
+  }
   if (done) {
     return DoneOp::generate();
   }
@@ -194,8 +222,8 @@ std::unique_ptr<IoOp> IoSequence::next() {
   return _next();
 }
 
-ceph::io_exerciser::Seq0::Seq0(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), offset(0) {
+ceph::io_exerciser::Seq0::Seq0(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), offset(0) {
   select_random_object_size();
   length = 1 + rng(obj_size - 1);
 }
@@ -226,8 +254,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq0::_next() {
   return r;
 }
 
-ceph::io_exerciser::Seq1::Seq1(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed) {
+ceph::io_exerciser::Seq1::Seq1(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency) {
   select_random_object_size();
   count = 3 * obj_size;
 }
@@ -258,8 +286,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq1::_next() {
   }
 }
 
-ceph::io_exerciser::Seq2::Seq2(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), offset(0), length(0) {}
+ceph::io_exerciser::Seq2::Seq2(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), offset(0), length(0) {}
 
 Sequence ceph::io_exerciser::Seq2::get_id() const {
   return Sequence::SEQUENCE_SEQ2;
@@ -283,8 +311,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq2::_next() {
   return SingleReadOp::generate(offset, length);
 }
 
-ceph::io_exerciser::Seq3::Seq3(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), offset1(0), offset2(0) {
+ceph::io_exerciser::Seq3::Seq3(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), offset1(0), offset2(0) {
   set_min_object_size(2);
 }
 
@@ -310,8 +338,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq3::_next() {
   return DoubleReadOp::generate(offset1, 1, offset1 + offset2, 1);
 }
 
-ceph::io_exerciser::Seq4::Seq4(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), offset1(0), offset2(1) {
+ceph::io_exerciser::Seq4::Seq4(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), offset1(0), offset2(1) {
   set_min_object_size(3);
 }
 
@@ -338,8 +366,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq4::_next() {
                                 (offset1 * 2 + offset2) / 2, 1);
 }
 
-ceph::io_exerciser::Seq5::Seq5(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed),
+ceph::io_exerciser::Seq5::Seq5(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency),
       offset(0),
       length(1),
       doneread(false),
@@ -358,6 +386,7 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq5::_next() {
     if (!doneread) {
       if (!donebarrier) {
         donebarrier = true;
+        consistency = check_consistency;
         return BarrierOp::generate();
       }
       doneread = true;
@@ -379,8 +408,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq5::_next() {
   return r;
 }
 
-ceph::io_exerciser::Seq6::Seq6(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed),
+ceph::io_exerciser::Seq6::Seq6(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency),
       offset(0),
       length(1),
       doneread(false),
@@ -399,6 +428,7 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq6::_next() {
     if (!doneread) {
       if (!donebarrier) {
         donebarrier = true;
+        consistency = check_consistency;
         return BarrierOp::generate();
       }
       doneread = true;
@@ -423,8 +453,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq6::_next() {
   return r;
 }
 
-ceph::io_exerciser::Seq7::Seq7(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed) {
+ceph::io_exerciser::Seq7::Seq7(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency) {
   set_min_object_size(2);
   offset = obj_size;
 }
@@ -441,6 +471,7 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq7::_next() {
   if (!doneread) {
     if (!donebarrier) {
       donebarrier = true;
+      consistency = check_consistency;
       return BarrierOp::generate();
     }
     doneread = true;
@@ -462,8 +493,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq7::_next() {
   return DoubleWriteOp::generate(offset, 1, obj_size / 2, 1);
 }
 
-ceph::io_exerciser::Seq8::Seq8(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), offset1(0), offset2(1) {
+ceph::io_exerciser::Seq8::Seq8(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), offset1(0), offset2(1) {
   set_min_object_size(3);
 }
 
@@ -479,6 +510,7 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq8::_next() {
   if (!doneread) {
     if (!donebarrier) {
       donebarrier = true;
+      consistency = check_consistency;
       return BarrierOp::generate();
     }
     doneread = true;
@@ -501,8 +533,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq8::_next() {
                                  (offset1 * 2 + offset2) / 2, 1);
 }
 
-ceph::io_exerciser::Seq9::Seq9(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), offset(0), length(0) {}
+ceph::io_exerciser::Seq9::Seq9(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), offset(0), length(0) {}
 
 Sequence ceph::io_exerciser::Seq9::get_id() const {
   return Sequence::SEQUENCE_SEQ9;
@@ -516,6 +548,7 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq9::_next() {
   if (!doneread) {
     if (!donebarrier) {
       donebarrier = true;
+      consistency = check_consistency;
       return BarrierOp::generate();
     }
     doneread = true;
@@ -537,8 +570,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq9::_next() {
   return SingleWriteOp::generate(offset, length);
 }
 
-ceph::io_exerciser::Seq11::Seq11(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed),
+ceph::io_exerciser::Seq11::Seq11(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency),
       count(0),
       doneread(false),
       donebarrier(false) {}
@@ -570,8 +603,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq11::_next() {
   return SingleAppendOp::generate(obj_size);
 }
 
-ceph::io_exerciser::Seq12::Seq12(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), count(0), overlap(1), doneread(false) {}
+ceph::io_exerciser::Seq12::Seq12(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), count(0), overlap(1), doneread(false) {}
 
 Sequence ceph::io_exerciser::Seq12::get_id() const {
   return Sequence::SEQUENCE_SEQ12;
@@ -606,8 +639,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq12::_next() {
                                  obj_size + overlap);
 }
 
-ceph::io_exerciser::Seq13::Seq13(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(obj_size_range, seed), count(0), gap(1), doneread(false) {
+ceph::io_exerciser::Seq13::Seq13(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(obj_size_range, seed, check_consistency), count(0), gap(1), doneread(false) {
   set_min_object_size(2);
 }
 
@@ -643,8 +676,8 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq13::_next() {
   return SingleWriteOp::generate((count * obj_size) + gap, obj_size - gap);
 }
 
-ceph::io_exerciser::Seq14::Seq14(std::pair<int, int> obj_size_range, int seed)
-    : IoSequence(std::make_pair(0, obj_size_range.second), seed),
+ceph::io_exerciser::Seq14::Seq14(std::pair<int, int> obj_size_range, int seed, bool check_consistency)
+    : IoSequence(std::make_pair(0, obj_size_range.second), seed, check_consistency),
       offset(0),
       step(1) {
   startrng = std::default_random_engine(seed);
@@ -676,6 +709,8 @@ std::string ceph::io_exerciser::Seq14::get_name() const {
 std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq14::_next() {
   if (offset >= target_obj_size) {
     if (!doneread) {
+      consistency = check_consistency;
+      barrier = check_consistency;
       doneread = true;
       return SingleReadOp::generate(0, current_size);
     }

diff --git a/src/common/io_exerciser/IoSequence.h b/src/common/io_exerciser/IoSequence.h
index c0680408db1342e76ba47e384b04bafdf15d4c26..86282cabcc1f71936481760f8c8cff92494f3429 100644 (file)
--- a/src/common/io_exerciser/IoSequence.h
+++ b/src/common/io_exerciser/IoSequence.h
@@ -74,7 +74,7 @@ class IoSequence {
 
   virtual bool is_supported(Sequence sequence) const;
   static std::unique_ptr<IoSequence> generate_sequence(
-      Sequence s, std::pair<int, int> obj_size_range, int seed);
+      Sequence s, std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
  protected:
   uint64_t min_obj_size;
@@ -83,13 +83,17 @@ class IoSequence {
   bool barrier;
   bool done;
   bool remove;
+  bool consistency;
+  bool consistency_in_progress;
+  bool consistency_request_sent;
+  bool check_consistency;
   uint64_t obj_size;
   int step;
   int seed;
   ceph::util::random_number_generator<int> rng =
       ceph::util::random_number_generator<int>();
 
-  IoSequence(std::pair<int, int> obj_size_range, int seed);
+  IoSequence(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   virtual std::unique_ptr<IoOp> _next() = 0;
 
@@ -97,11 +101,12 @@ class IoSequence {
   void set_max_object_size(uint64_t size);
   void select_random_object_size();
   std::unique_ptr<IoOp> increment_object_size();
+  std::unique_ptr<IoOp> process_remove();
 };
 
 class Seq0 : public IoSequence {
  public:
-  Seq0(std::pair<int, int> obj_size_range, int seed);
+  Seq0(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -114,7 +119,7 @@ class Seq0 : public IoSequence {
 
 class Seq1 : public IoSequence {
  public:
-  Seq1(std::pair<int, int> obj_size_range, int seed);
+  Seq1(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -126,7 +131,7 @@ class Seq1 : public IoSequence {
 
 class Seq2 : public IoSequence {
  public:
-  Seq2(std::pair<int, int> obj_size_range, int seed);
+  Seq2(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -139,7 +144,7 @@ class Seq2 : public IoSequence {
 
 class Seq3 : public IoSequence {
  public:
-  Seq3(std::pair<int, int> obj_size_range, int seed);
+  Seq3(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -152,7 +157,7 @@ class Seq3 : public IoSequence {
 
 class Seq4 : public IoSequence {
  public:
-  Seq4(std::pair<int, int> obj_size_range, int seed);
+  Seq4(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -165,7 +170,7 @@ class Seq4 : public IoSequence {
 
 class Seq5 : public IoSequence {
  public:
-  Seq5(std::pair<int, int> obj_size_range, int seed);
+  Seq5(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -180,7 +185,7 @@ class Seq5 : public IoSequence {
 
 class Seq6 : public IoSequence {
  public:
-  Seq6(std::pair<int, int> obj_size_range, int seed);
+  Seq6(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -195,7 +200,7 @@ class Seq6 : public IoSequence {
 
 class Seq7 : public IoSequence {
  public:
-  Seq7(std::pair<int, int> obj_size_range, int seed);
+  Seq7(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -209,7 +214,7 @@ class Seq7 : public IoSequence {
 
 class Seq8 : public IoSequence {
  public:
-  Seq8(std::pair<int, int> obj_size_range, int seed);
+  Seq8(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -230,7 +235,7 @@ class Seq9 : public IoSequence {
   bool donebarrier = false;
 
  public:
-  Seq9(std::pair<int, int> obj_size_range, int seed);
+  Seq9(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
       Sequence get_id() const override;
       std::string get_name() const override;
@@ -244,7 +249,7 @@ class Seq11 : public IoSequence {
   bool donebarrier = false;
 
  public:
-  Seq11(std::pair<int, int> obj_size_range, int seed);
+  Seq11(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -259,7 +264,7 @@ class Seq12 : public IoSequence {
   bool donebarrier = false;
 
  public:
-  Seq12(std::pair<int, int> obj_size_range, int seed);
+  Seq12(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -274,7 +279,7 @@ class Seq13 : public IoSequence {
   bool donebarrier = false;
 
  public:
-  Seq13(std::pair<int, int> obj_size_range, int seed);
+  Seq13(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   Sequence get_id() const override;
   std::string get_name() const override;
@@ -293,7 +298,7 @@ class Seq14 : public IoSequence {
   bool doneread = false;
 
  public:
-  Seq14(std::pair<int, int> obj_size_range, int seed);
+  Seq14(std::pair<int, int> obj_size_range, int seed, bool check_consistency);
 
   void setup_starts();
   Sequence get_id() const override;

diff --git a/src/common/io_exerciser/OpType.h b/src/common/io_exerciser/OpType.h
index 6a7e4604407e5590ae0ced730594e3487b75e458..3c7f535839c31aad309b498438a4f11db9c89f60 100644 (file)
--- a/src/common/io_exerciser/OpType.h
+++ b/src/common/io_exerciser/OpType.h
@@ -17,6 +17,7 @@ enum class OpType {
   Barrier,               // Barrier - all prior I/Os must complete
   Create,                // Create object and pattern with data
   Remove,                // Remove object
+  Consistency,           // Check consistency of an object
   Read,                  // Read
   Read2,                 // Two reads in a single op
   Read3,                 // Three reads in a single op
@@ -59,6 +60,8 @@ struct fmt::formatter<ceph::io_exerciser::OpType> {
         return fmt::format_to(ctx.out(), "Create");
       case ceph::io_exerciser::OpType::Remove:
         return fmt::format_to(ctx.out(), "Remove");
+      case ceph::io_exerciser::OpType::Consistency:
+        return fmt::format_to(ctx.out(), "Consistency");
       case ceph::io_exerciser::OpType::Read:
         return fmt::format_to(ctx.out(), "Read");
       case ceph::io_exerciser::OpType::Read2:

diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc
index e411ddede4907dafc982861d349a0de22179b13f..1b07d4d12ae15e6eac8ce56fd9b4d2915f279631 100644 (file)
--- a/src/common/io_exerciser/RadosIo.cc
+++ b/src/common/io_exerciser/RadosIo.cc
@@ -10,6 +10,7 @@
 #include "common/json/OSDStructures.h"
 
 using RadosIo = ceph::io_exerciser::RadosIo;
+using ConsistencyChecker = ceph::consistency::ConsistencyChecker;
 
 namespace {
 template <typename S>
@@ -48,6 +49,7 @@ RadosIo::RadosIo(librados::Rados& rados, boost::asio::io_context& asio,
       om(std::make_unique<ObjectModel>(oid, block_size, seed)),
       db(data_generation::DataGenerator::create_generator(
           data_generation::GenerationType::HeaderedSeededRandom, *om)),
+      cc(std::make_unique<ConsistencyChecker>(rados, asio, pool)),
       pool(pool),
       cached_shard_order(cached_shard_order),
       threads(threads),
@@ -188,6 +190,16 @@ void RadosIo::applyIoOp(IoOp& op) {
                               std::move(wop), 0, nullptr, remove_cb);
       break;
     }
+
+    case OpType::Consistency: {
+      start_io();
+      bool is_consistent =
+          cc->single_read_and_check_consistency(oid, block_size, 0, 0);
+      ceph_assert(is_consistent);
+      finish_io();
+      break;
+    }
+
     case OpType::Read:
       [[fallthrough]];
     case OpType::Read2:
@@ -496,4 +508,4 @@ void RadosIo::applyInjectOp(IoOp& op) {
           fmt::format("Unsupported inject operation ({})", op.getOpType()));
       break;
   }
-}
+}
\ No newline at end of file

diff --git a/src/common/io_exerciser/RadosIo.h b/src/common/io_exerciser/RadosIo.h
index 1837b75725ae0d56b69c1d3e0c19ddeb4b722f46..c9add85bb82dcc9287f06634333f3216b99c4fbf 100644 (file)
--- a/src/common/io_exerciser/RadosIo.h
+++ b/src/common/io_exerciser/RadosIo.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "ObjectModel.h"
+#include "erasure-code/consistency/ConsistencyChecker.h"
 
 /* Overview
  *
@@ -25,6 +26,7 @@ class RadosIo : public Model {
   boost::asio::io_context& asio;
   std::unique_ptr<ObjectModel> om;
   std::unique_ptr<ceph::io_exerciser::data_generation::DataGenerator> db;
+  std::unique_ptr<ceph::consistency::ConsistencyChecker> cc;
   std::string pool;
   std::optional<std::vector<int>> cached_shard_order;
   int threads;

diff --git a/src/test/osd/CMakeLists.txt b/src/test/osd/CMakeLists.txt
index a777ee9179aa355ae5d1b1d4869dcbdb9ac54334..cbe5113d3829fb9fedf86a55777e33186dd865a0 100644 (file)
--- a/src/test/osd/CMakeLists.txt
+++ b/src/test/osd/CMakeLists.txt
@@ -23,7 +23,7 @@ add_executable(ceph_test_rados_io_sequence
   ${CMAKE_CURRENT_SOURCE_DIR}/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc)
 add_dependencies(ceph_test_rados_io_sequence erasure_code_plugins)
 target_link_libraries(ceph_test_rados_io_sequence
-  librados global object_io_exerciser json_structures)
+  librados global object_io_exerciser json_structures ec_consistency)
 install(TARGETS
   ceph_test_rados_io_sequence
   DESTINATION ${CMAKE_INSTALL_BINDIR})

diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc
index 396ec815afaff5dee6274346866c18a5f1f50a83..a99f3de674c458e81067369ad77361c2e3092fdd 100644 (file)
--- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc
+++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc
@@ -195,6 +195,8 @@ po::options_description get_options_description() {
       "number of objects to exercise in parallel")(
       "testrecovery",
       "Inject errors during sequences to test recovery processes of OSDs")(
+      "checkconsistency",
+      "Test objects for consistency during IO sequences. Disabled by default.")(
       "interactive", "interactive mode, execute IO commands from stdin")(
       "allow_pool_autoscaling",
       "Allows pool autoscaling. Disabled by default.")(
@@ -947,8 +949,9 @@ ceph::io_sequence::tester::TestObject::TestObject(
     SelectObjectSize& sos, SelectNumThreads& snt, SelectSeqRange& ssr,
     ceph::util::random_number_generator<int>& rng, ceph::mutex& lock,
     ceph::condition_variable& cond, bool dryrun, bool verbose,
-    std::optional<int> seqseed, bool testrecovery)
-    : rng(rng), verbose(verbose), seqseed(seqseed), testrecovery(testrecovery) {
+    std::optional<int> seqseed, bool testrecovery, bool checkconsistency)
+    : rng(rng), verbose(verbose), seqseed(seqseed),
+      testrecovery(testrecovery), checkconsistency(checkconsistency) {
   if (dryrun) {
     exerciser_model = std::make_unique<ceph::io_exerciser::ObjectModel>(
         oid, sbs.select(), rng());
@@ -1001,10 +1004,10 @@ ceph::io_sequence::tester::TestObject::TestObject(
   if (testrecovery) {
     seq = ceph::io_exerciser::EcIoSequence::generate_sequence(
         curseq, obj_size_range, pool_km, pool_mappinglayers,
-        seqseed.value_or(rng()));
+        seqseed.value_or(rng()), checkconsistency);
   } else {
     seq = ceph::io_exerciser::IoSequence::generate_sequence(
-        curseq, obj_size_range, seqseed.value_or(rng()));
+        curseq, obj_size_range, seqseed.value_or(rng()), checkconsistency);
   }
 
   op = seq->next();
@@ -1040,10 +1043,10 @@ bool ceph::io_sequence::tester::TestObject::next() {
         if (testrecovery) {
           seq = ceph::io_exerciser::EcIoSequence::generate_sequence(
               curseq, obj_size_range, pool_km, pool_mappinglayers,
-              seqseed.value_or(rng()));
+              seqseed.value_or(rng()), checkconsistency);
         } else {
           seq = ceph::io_exerciser::IoSequence::generate_sequence(
-              curseq, obj_size_range, seqseed.value_or(rng()));
+              curseq, obj_size_range, seqseed.value_or(rng()), checkconsistency);
         }
 
         dout(0) << "== " << exerciser_model->get_oid() << " " << curseq << " "
@@ -1098,6 +1101,7 @@ ceph::io_sequence::tester::TestRunner::TestRunner(
   object_name = vm["object"].as<std::string>();
   interactive = vm.contains("interactive");
   testrecovery = vm.contains("testrecovery");
+  checkconsistency = vm.contains("checkconsistency");
 
   allow_pool_autoscaling = vm.contains("allow_pool_autoscaling");
   allow_pool_balancer = vm.contains("allow_pool_balancer");
@@ -1131,7 +1135,7 @@ void ceph::io_sequence::tester::TestRunner::help() {
 }
 
 void ceph::io_sequence::tester::TestRunner::list_sequence(bool testrecovery) {
-  // List seqeunces
+  // List sequences
   std::pair<int, int> obj_size_range = sos.select();
   ceph::io_exerciser::Sequence s = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN;
   std::unique_ptr<ceph::io_exerciser::IoSequence> seq;
@@ -1151,11 +1155,11 @@ void ceph::io_sequence::tester::TestRunner::list_sequence(bool testrecovery) {
   do {
     if (testrecovery) {
       seq = ceph::io_exerciser::EcIoSequence::generate_sequence(
-        s, obj_size_range, km, mappinglayers, seqseed.value_or(rng()));
+        s, obj_size_range, km, mappinglayers, seqseed.value_or(rng()), checkconsistency);
     }
     else {
       seq = ceph::io_exerciser::IoSequence::generate_sequence(
-        s, obj_size_range, seqseed.value_or(rng()));
+        s, obj_size_range, seqseed.value_or(rng()), checkconsistency);
     }
 
     dout(0) << s << " " << seq->get_name_with_seqseed() << dendl;
@@ -1413,7 +1417,7 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() {
       test_objects.push_back(
           std::make_shared<ceph::io_sequence::tester::TestObject>(
               name, rados, asio, sbs, spo, sos, snt, ssr, rng, lock, cond,
-              dryrun, verbose, seqseed, testrecovery));
+              dryrun, verbose, seqseed, testrecovery, checkconsistency));
     }
     catch (const std::runtime_error &e) {
       std::cerr << "Error: " << e.what() << std::endl;

diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h
index 16fe6d7b778d1bbae798ee5b3b4b9728ffc8bf18..26f2cd5d9e0a161fba6b208d5976f7f877c66edc 100644 (file)
--- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h
+++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h
@@ -443,7 +443,8 @@ class TestObject {
              bool dryrun,
              bool verbose,
              std::optional<int> seqseed,
-             bool testRecovery);
+             bool testRecovery,
+             bool checkConsistency);
 
   int get_num_io();
   bool readyForIo();
@@ -466,6 +467,7 @@ class TestObject {
   std::optional<std::pair<std::string_view, std::string_view>>
       pool_mappinglayers;
   bool testrecovery;
+  bool checkconsistency;
 };
 
 class TestRunner {
@@ -504,6 +506,7 @@ class TestRunner {
   bool interactive;
 
   bool testrecovery;
+  bool checkconsistency;
 
   bool allow_pool_autoscaling;
   bool allow_pool_balancer;