test/osd: Unit test to verify optimised EC deep scrub functionality

Signed-off-by: Jon Bailey <jonathan.bailey1@ibm.com>

diff --git a/src/test/osd/scrubber_generators.cc b/src/test/osd/scrubber_generators.cc
index 77e6cf43acc5c4cdc364f8db90c99b63779031c9..5802053a3b7b7a7d4ae09220a14d06c824f12539 100644 (file)
--- a/src/test/osd/scrubber_generators.cc
+++ b/src/test/osd/scrubber_generators.cc
@@ -5,6 +5,8 @@
 
 #include <fmt/ranges.h>
 
+#include "scrubber_test_datasets.h"
+
 using namespace ScrubGenerator;
 
 // ref: PGLogTestRebuildMissing()
@@ -37,6 +39,28 @@ std::pair<bufferlist, std::vector<snapid_t>> create_object_snapset(
   return {bl, sns.clones};
 }
 
+RealObjsConf ScrubGenerator::make_erasure_code_configuration(int8_t k,
+                                                             int8_t m) {
+  RealObjsConf erasure_code_configuration;
+  for (shard_id_t i{0}; i < k + m; ++i) {
+    RealObj erasure_code_obj = ScrubDatasets::erasure_code_obj;
+    erasure_code_obj.ghobj.shard_id = i;
+    erasure_code_configuration.objs.push_back(erasure_code_obj);
+  }
+
+  return erasure_code_configuration;
+}
+
+CorruptFuncList ScrubGenerator::make_erasure_code_hash_corruption_functions(
+    int num_osds) {
+  CorruptFuncList ret_list = {};
+  for (int i = 0; i < num_osds; ++i) {
+    ret_list.insert({i, &crpt_object_hash});
+  }
+
+  return ret_list;
+}
+
 RealObjsConfList ScrubGenerator::make_real_objs_conf(
   int64_t pool_id,
   const RealObjsConf& blueprint,

diff --git a/src/test/osd/scrubber_generators.h b/src/test/osd/scrubber_generators.h
index d21511162900f5f5b6fe6231f4b0caace6e009f6..4fde8a7f444dc51236be36a9ef7cf0c5171d5503 100644 (file)
--- a/src/test/osd/scrubber_generators.h
+++ b/src/test/osd/scrubber_generators.h
@@ -210,6 +210,13 @@ static inline RealObj crpt_do_nothing(const RealObj& s, int osdn)
   return s;
 }
 
+static inline RealObj crpt_object_hash(const RealObj& s,
+                                       [[maybe_unused]] int osdn) {
+  RealObj ret = s;
+  ret.data.hash = s.data.hash + 1;
+  return ret;
+}
+
 struct SmapEntry {
   ghobject_t ghobj;
   ScrubMap::object smobj;
@@ -230,6 +237,10 @@ struct RealObjsConf {
   std::vector<RealObj> objs;
 };
 
+RealObjsConf make_erasure_code_configuration(int8_t k, int8_t m);
+
+CorruptFuncList make_erasure_code_hash_corruption_functions(int num_osds);
+
 using RealObjsConfRef = std::unique_ptr<RealObjsConf>;
 
 // RealObjsConf will be "developed" into the following of per-osd sets,

diff --git a/src/test/osd/scrubber_test_datasets.cc b/src/test/osd/scrubber_test_datasets.cc
index 51be0fb5085957746ffe2616bf9719084e921390..243f045e20986a3723a8005008adb86bd5a1ec23 100644 (file)
--- a/src/test/osd/scrubber_test_datasets.cc
+++ b/src/test/osd/scrubber_test_datasets.cc
@@ -39,6 +39,13 @@ static hobject_t hobj_ms1{object_t{"hobj_ms1"},
                          0,            // pool
                          ""s};         // nspace
 
+hobject_t ec_hobj_ms1{object_t{"ec_hobj_ms1"},
+                      "keykey",     // key
+                      CEPH_NOSNAP,  // snap_id
+                      0,            // hash
+                      0,            // pool
+                      ""s};         // nspace
+
 SnapsetMockData::CookedCloneSnaps ms1_fn()
 {
   std::map<snapid_t, uint64_t> clnsz;
@@ -55,10 +62,15 @@ SnapsetMockData::CookedCloneSnaps ms1_fn()
   return {clnsz, clnsn, overlaps};
 }
 
+SnapsetMockData::CookedCloneSnaps ms2_fn() { return {{}, {}, {}}; }
+
 static SnapsetMockData hobj_ms1_snapset{/* seq */ 0x40,
                                        /* clones */ {0x20, 0x30},
                                        ms1_fn};
 
+static SnapsetMockData empty_snapset{/* seq */ 0x0,
+                                     /* clones */ {}, ms2_fn};
+
 hobject_t hobj_ms1_snp30{object_t{"hobj_ms1"},
                         "keykey",  // key
                         0x30,      // snap_id
@@ -116,4 +128,8 @@ ScrubGenerator::RealObjsConf minimal_snaps_configuration{
 
 };
 
+ScrubGenerator::RealObj erasure_code_obj{
+    ghobject_t{ec_hobj_ms1, 1, shard_id_t{0}},
+    RealData{100, 0xf8346009, 0, 0, {}, {}}, &crpt_funcs_set0, &empty_snapset};
+
 }  // namespace ScrubDatasets

diff --git a/src/test/osd/scrubber_test_datasets.h b/src/test/osd/scrubber_test_datasets.h
index 18152856830337fd7dbebc366885bbc9208c43a0..fe2d1551f514c041fbf89efb5209e1670df2c624 100644 (file)
--- a/src/test/osd/scrubber_test_datasets.h
+++ b/src/test/osd/scrubber_test_datasets.h
@@ -12,6 +12,11 @@ namespace ScrubDatasets {
  */
 extern ScrubGenerator::RealObjsConf minimal_snaps_configuration;
 
+/*
+ * Dataset to represent an erasure coded configuration.
+ */
+extern ScrubGenerator::RealObj erasure_code_obj;
+
 // and a part of this configuration, one that we will corrupt in a test:
 extern hobject_t hobj_ms1_snp30;
 

diff --git a/src/test/osd/test_scrubber_be.cc b/src/test/osd/test_scrubber_be.cc
index 94ee6ef843bf53a9a1ebd87d15b4f6879d4f703c..bc6db59ac17af663ea971b0c49f9ce290886c0ea 100644 (file)
--- a/src/test/osd/test_scrubber_be.cc
+++ b/src/test/osd/test_scrubber_be.cc
@@ -24,6 +24,8 @@
 #include "osd/scrubber/pg_scrubber.h"
 #include "osd/scrubber/scrub_backend.h"
 
+#include "erasure-code/ErasureCodePlugin.h"
+
 /// \file testing isolated parts of the Scrubber backend
 
 using namespace std::string_literals;
@@ -114,6 +116,7 @@ class TestPg : public PgScrubBeListener {
       bl.rebuild();
       encode_map.insert(i, bl);
     }
+
     for (shard_id_t i; i < get_ec_sinfo().get_k(); ++i) {
       for (int j = 0; j < get_ec_sinfo().get_chunk_size(); j++) {
         encode_map.at(i).c_str()[j] =
@@ -802,6 +805,132 @@ TEST_F(TestTScrubberBe_data_2, smaps_clone_size)
   EXPECT_EQ(incons.size(), 1); // one inconsistency
 }
 
+class TestTScrubberBeECCorruptShards : public TestTScrubberBe {
+ private:
+  int seed;
+
+ protected:
+  std::mt19937 rng;
+  int8_t k;
+  int8_t m;
+  int m_chunk_size = 4;
+
+ public:
+  TestTScrubberBeECCorruptShards()
+      : TestTScrubberBe(),
+        seed(time(0)),
+        rng(seed),
+        k((rng() % 10) + 2),
+        m((rng() % std::min(k - 1, 4)) + 1) {
+    std::cout << "Using seed " << seed << std::endl;
+  }
+
+  std::size_t m_expected_inconsistencies = 0;
+  std::size_t m_expected_error_count = 0;
+
+  // basic test configuration - 3 OSDs, all involved in the pool
+  erasure_code_profile_conf_t ec_profile{
+      "scrub_ec_profile",
+      {{"k", fmt::format("{}", k)},
+       {"m", fmt::format("{}", m)},
+       {"plugin", "isa"},
+       {"technique", "reed_sol_van"},
+       {"stripe_unit", fmt::format("{}", m_chunk_size)}}};
+
+  pool_conf_t pl{3,         3, k + m, k + 1, "ec_pool", pg_pool_t::TYPE_ERASURE,
+                 ec_profile};
+
+  TestTScrubberBeParams inject_params() override {
+    std::cout << fmt::format(
+                     "{}: injecting params (minimal-snaps + size change)",
+                     __func__)
+              << std::endl;
+    TestTScrubberBeParams params{
+        /* pool_conf */ pl,
+        /* real_objs_conf */
+        ScrubGenerator::make_erasure_code_configuration(k, m),
+        /*num_osds */ k + m};
+
+    return params;
+  }
+};
+
+class TestTScrubberBeECNoCorruptShards : public TestTScrubberBeECCorruptShards {
+ public:
+  TestTScrubberBeECNoCorruptShards() : TestTScrubberBeECCorruptShards() {}
+};
+
+TEST_F(TestTScrubberBeECNoCorruptShards, ec_parity_inconsistency) {
+  test_pg->set_stripe_info(k, m, k * m_chunk_size, &test_pg->m_pool->info);
+
+  ASSERT_TRUE(sbe);  // Assert we have a scrubber backend
+  logger.set_expected_err_count(
+      0);  // Set the number of errors we expect to see
+
+  auto [incons, fix_list] = sbe->scrub_compare_maps(true, *test_scrubber);
+
+  EXPECT_EQ(incons.size(), 0);
+}
+
+class TestTScrubberBeECSingleCorruptDataShard
+    : public TestTScrubberBeECCorruptShards {
+ public:
+  TestTScrubberBeECSingleCorruptDataShard()
+      : TestTScrubberBeECCorruptShards() {}
+
+  TestTScrubberBeParams inject_params() override {
+    TestTScrubberBeParams params =
+        TestTScrubberBeECCorruptShards::inject_params();
+    corrupt_funcs = make_erasure_code_hash_corruption_functions(k + m);
+    params.objs_conf.objs[(rng() % k)].corrupt_funcs = &corrupt_funcs;
+    return params;
+  }
+
+ private:
+  CorruptFuncList corrupt_funcs;
+};
+
+TEST_F(TestTScrubberBeECSingleCorruptDataShard, ec_parity_inconsistency) {
+  test_pg->set_stripe_info(k, m, k * m_chunk_size, &test_pg->m_pool->info);
+
+  ASSERT_TRUE(sbe);  // Assert we have a scrubber backend
+  logger.set_expected_err_count(
+      1);  // Set the number of errors we expect to see
+
+  auto [incons, fix_list] = sbe->scrub_compare_maps(true, *test_scrubber);
+
+  EXPECT_EQ(incons.size(), 1);
+}
+
+class TestTScrubberBeECCorruptParityShard
+    : public TestTScrubberBeECCorruptShards {
+ public:
+  TestTScrubberBeECCorruptParityShard() : TestTScrubberBeECCorruptShards() {}
+
+  TestTScrubberBeParams inject_params() override {
+    TestTScrubberBeParams params =
+        TestTScrubberBeECCorruptShards::inject_params();
+    corrupt_funcs = make_erasure_code_hash_corruption_functions(k + m);
+    params.objs_conf.objs[k].corrupt_funcs = &corrupt_funcs;
+    return params;
+  }
+
+ private:
+  CorruptFuncList corrupt_funcs;
+};
+
+TEST_F(TestTScrubberBeECCorruptParityShard, ec_parity_inconsistency) {
+  test_pg->set_stripe_info(k, m, k * m_chunk_size, &test_pg->m_pool->info);
+
+  ASSERT_TRUE(sbe);  // Assert we have a scrubber backend
+  logger.set_expected_err_count(
+      1);  // Set the number of errors we expect to see
+
+  auto [incons, fix_list] = sbe->scrub_compare_maps(true, *test_scrubber);
+
+  EXPECT_EQ(incons.size(), 1);
+}
+
 // Local Variables:
 // compile-command: "cd ../.. ; make unittest_osdscrub ; ./unittest_osdscrub
 // --log-to-stderr=true  --debug-osd=20 # --gtest_filter=*.* " End: