From 4022d6e92ee58c0ba68280d40cfec015dd91c43e Mon Sep 17 00:00:00 2001 From: Jon Bailey Date: Thu, 8 May 2025 13:25:58 +0100 Subject: [PATCH] test/osd: Add erasure coded pools support to test_scrubber_be.cc Signed-off-by: Jon Bailey --- src/test/osd/scrubber_generators.cc | 24 ++++- src/test/osd/scrubber_generators.h | 13 ++- src/test/osd/test_scrubber_be.cc | 135 ++++++++++++++++++++++++---- 3 files changed, 149 insertions(+), 23 deletions(-) diff --git a/src/test/osd/scrubber_generators.cc b/src/test/osd/scrubber_generators.cc index 94e2ba46e9ba4..77e6cf43acc5c 100644 --- a/src/test/osd/scrubber_generators.cc +++ b/src/test/osd/scrubber_generators.cc @@ -40,16 +40,35 @@ std::pair> create_object_snapset( RealObjsConfList ScrubGenerator::make_real_objs_conf( int64_t pool_id, const RealObjsConf& blueprint, - std::vector active_osds) + std::vector active_osds, + std::set acting_shards, + bool erasure_coded_pool) { RealObjsConfList all_osds; for (auto osd : active_osds) { + shard_id_t shard = std::find_if(acting_shards.begin(), acting_shards.end(), + [&osd](pg_shard_t pg_shard) { + return osd == pg_shard.osd; + }) + ->shard; + RealObjsConfRef this_osd_fakes = std::make_unique(blueprint); // now - fix & corrupt every "object" in the blueprint for (RealObj& robj : this_osd_fakes->objs) { + if (!erasure_coded_pool || robj.ghobj.shard_id == shard) { + robj.ghobj.hobj.pool = pool_id; + } + } - robj.ghobj.hobj.pool = pool_id; + if (erasure_coded_pool) { + this_osd_fakes->objs.erase(std::remove_if(this_osd_fakes->objs.begin(), + this_osd_fakes->objs.end(), + [pool_id](RealObj& robj) { + return robj.ghobj.hobj.pool != + pool_id; + }), + this_osd_fakes->objs.end()); } all_osds[osd] = std::move(this_osd_fakes); @@ -92,6 +111,7 @@ ScrubGenerator::SmapEntry ScrubGenerator::make_smobject( } ret.smobj.size = blueprint.data.size; ret.smobj.digest = blueprint.data.hash; + ret.smobj.digest_present = true; /// \todo handle the 'present' etc' ret.smobj.object_omap_keys = blueprint.data.omap.size(); diff --git a/src/test/osd/scrubber_generators.h b/src/test/osd/scrubber_generators.h index 47f63e52401b6..d21511162900f 100644 --- a/src/test/osd/scrubber_generators.h +++ b/src/test/osd/scrubber_generators.h @@ -103,12 +103,19 @@ class MockLog : public LoggerSinkSet { // ///////////////////////////////////////////////////////////////////////// // // ///////////////////////////////////////////////////////////////////////// // +struct erasure_code_profile_conf_t { + std::string erasure_code_profile_name{"erasure_code_profile"}; + std::map erasure_code_profile{}; +}; + struct pool_conf_t { int pg_num{3}; int pgp_num{3}; int size{3}; int min_size{3}; std::string name{"rep_pool"}; + uint8_t type{pg_pool_t::TYPE_REPLICATED}; + std::optional erasure_code_profile; }; using attr_t = std::map; @@ -231,8 +238,10 @@ using RealObjsConfRef = std::unique_ptr; using RealObjsConfList = std::map; RealObjsConfList make_real_objs_conf(int64_t pool_id, - const RealObjsConf& blueprint, - std::vector active_osds); + const RealObjsConf& blueprint, + std::vector active_osds, + std::set active_shards, + bool erasure_coded_pool); /** * create the snap-ids set for all clones appearing in the head diff --git a/src/test/osd/test_scrubber_be.cc b/src/test/osd/test_scrubber_be.cc index ef154eca4e6d3..94ee6ef843bf5 100644 --- a/src/test/osd/test_scrubber_be.cc +++ b/src/test/osd/test_scrubber_be.cc @@ -71,6 +71,8 @@ class TestScrubBackend : public ScrubBackend { // mocking the PG class TestPg : public PgScrubBeListener { + ErasureCodeInterfaceRef m_erasure_code_interface; + public: ~TestPg() = default; @@ -97,18 +99,80 @@ class TestPg : public PgScrubBeListener { } bool ec_can_decode(const shard_id_set& available_shards) const final { - return false; + return get_is_ec_optimized() && + available_shards.size() > get_ec_sinfo().get_k_plus_m() - 2; }; + // Fake encode function for erasure code tests in this class. + // Just sets parities to sum of data shards. shard_id_map ec_encode_acting_set( - const bufferlist& in_bl) const final { - return shard_id_map(0); + const bufferlist& chunks) const final { + shard_id_map encode_map(get_ec_sinfo().get_k_plus_m()); + for (shard_id_t i; i < get_ec_sinfo().get_k_plus_m(); ++i) { + bufferlist bl; + bl.append(buffer::create(get_ec_sinfo().get_chunk_size(), 0)); + bl.rebuild(); + encode_map.insert(i, bl); + } + for (shard_id_t i; i < get_ec_sinfo().get_k(); ++i) { + for (int j = 0; j < get_ec_sinfo().get_chunk_size(); j++) { + encode_map.at(i).c_str()[j] = + chunks[j + (get_ec_sinfo().get_chunk_size() * i.id)]; + for (shard_id_t k{static_cast(get_ec_sinfo().get_k_plus_m())}; + k < get_ec_sinfo().get_k_plus_m(); ++k) { + encode_map.at(k).c_str()[j] += + chunks[j + (get_ec_sinfo().get_chunk_size() * i.id)]; + } + } + } + + return encode_map; }; + // Fake encode function for erasure code tests in this class. + // Just sets calculates missing parity by using sum of all data shards = + // parity Tests using this will only have 1 missing shard. shard_id_map ec_decode_acting_set( const shard_id_map& chunks, int chunk_size) const final { - return shard_id_map(0); - } + shard_id_map decode_map(get_ec_sinfo().get_k_plus_m()); + + ceph_assert(chunks.size() > get_ec_sinfo().get_k_plus_m() - 2); + + for (shard_id_t i; i < get_ec_sinfo().get_k_plus_m(); ++i) { + bufferlist bl; + bufferptr ptr = buffer::create(chunk_size, 0); + if (chunks.contains(i)) { + ptr.copy_in(0, chunk_size, chunks.at(i).to_str().c_str()); + } + bl.append(ptr); + bl.rebuild(); + decode_map.insert(i, bl); + } + + for (shard_id_t shard; shard < get_ec_sinfo().get_k_plus_m(); ++shard) { + if (!chunks.contains(shard)) { + for (int j = 0; j < chunk_size; j++) { + if (shard < get_ec_sinfo().get_k()) { + decode_map.at(shard).c_str()[j] = + decode_map + .at(shard_id_t{static_cast(get_ec_sinfo().get_k())}) + .c_str()[j]; + } + for (shard_id_t i; i < get_ec_sinfo().get_k(); ++i) { + if (shard < get_ec_sinfo().get_k() && chunks.contains(i)) { + decode_map.at(shard).c_str()[j] -= decode_map.at(i).c_str()[j]; + } else if (chunks.contains(i)) { + decode_map.at(shard).c_str()[j] += decode_map.at(i).c_str()[j]; + } + } + } + } else { + decode_map.insert(shard, chunks.at(shard)); + } + } + + return decode_map; + }; bool get_ec_supports_crc_encode_decode() const final { return get_is_ec_optimized(); @@ -305,7 +369,8 @@ class TestTScrubberBe : public ::testing::Test { * generated by the Primary). Then - create the snap-sets for all * the objects in the set. */ - void fake_a_scrub_set(ScrubGenerator::RealObjsConfList& all_sets); + void fake_a_scrub_set(ScrubGenerator::RealObjsConfList& all_sets, + std::set acting_shards); std::unique_ptr sbe; @@ -382,7 +447,12 @@ void TestTScrubberBe::SetUp() std::cout << fmt::format("PG info: {}", info) << std::endl; real_objs_list = - ScrubGenerator::make_real_objs_conf(pool_id, real_objs, acting_osds); + ScrubGenerator::make_real_objs_conf(pool_id, + real_objs, + acting_osds, + acting_shards, + pool_conf + .erasure_code_profile.has_value()); // now we can create the main mockers @@ -407,7 +477,7 @@ void TestTScrubberBe::SetUp() } sbe->new_chunk(); - fake_a_scrub_set(real_objs_list); + fake_a_scrub_set(real_objs_list, acting_shards); } @@ -436,7 +506,7 @@ OSDMapRef TestTScrubberBe::setup_map(int num_osds, } osdmap->apply_incremental(pending_inc); - // create a replicated pool + // create a pool OSDMap::Incremental new_pool_inc(osdmap->get_epoch() + 1); new_pool_inc.new_pool_max = osdmap->get_pool_max(); new_pool_inc.fsid = osdmap->get_fsid(); @@ -444,12 +514,22 @@ OSDMapRef TestTScrubberBe::setup_map(int num_osds, pg_pool_t empty; auto p = new_pool_inc.get_new_pool(pool_id, &empty); p->size = pconf.size; + p->min_size = pconf.min_size; p->set_pg_num(pconf.pg_num); p->set_pgp_num(pconf.pgp_num); - p->type = pg_pool_t::TYPE_REPLICATED; + p->type = pconf.type; p->crush_rule = 0; p->set_flag(pg_pool_t::FLAG_HASHPSPOOL); new_pool_inc.new_pool_names[pool_id] = pconf.name; + if (pconf.erasure_code_profile) { + osdmap->set_erasure_code_profile( + pconf.erasure_code_profile->erasure_code_profile_name, + pconf.erasure_code_profile->erasure_code_profile); + p->erasure_code_profile = + pconf.erasure_code_profile->erasure_code_profile_name; + p->set_flag(pg_pool_t::FLAG_EC_OVERWRITES); + p->set_flag(pg_pool_t::FLAG_EC_OPTIMIZATIONS); + } osdmap->apply_incremental(new_pool_inc); return osdmap; } @@ -480,17 +560,26 @@ pg_info_t TestTScrubberBe::setup_pg_in_map() << std::endl; spg = spg_t{pgid}; - i_am = pg_shard_t{up_primary}; - std::cout << fmt::format("{}: spg: {} and I am {}", __func__, spg, i_am) - << std::endl; // the 'acting shards' set - the one actually used by the scrubber + shard_id_t shard; std::for_each(acting_osds.begin(), acting_osds.end(), [&](int osd) { - acting_shards.insert(pg_shard_t{osd}); + acting_shards.insert(pg_shard_t{osd, shard}); + ++shard; }); std::cout << fmt::format("{}: acting_shards: {}", __func__, acting_shards) << std::endl; + shard_id_t osd_shard = + std::find_if( + acting_shards.begin(), acting_shards.end(), + [&](pg_shard_t pg_shard) { return i_am.osd == pg_shard.osd; }) + ->shard; + + i_am = pg_shard_t{up_primary, osd_shard}; + std::cout << fmt::format("{}: spg: {} and I am {}", __func__, spg, i_am) + << std::endl; + pg_info_t info; info.pgid = spg; /// \todo: handle the epochs: @@ -514,8 +603,8 @@ void TestTScrubberBe::TearDown() } void TestTScrubberBe::fake_a_scrub_set( - ScrubGenerator::RealObjsConfList& all_sets) -{ + ScrubGenerator::RealObjsConfList& all_sets, + std::set acting_shards) { for (int osd_num = 0; osd_num < pool_conf.size; ++osd_num) { ScrubMap smap; smap.valid_through = eversion_t{1, 1}; @@ -529,12 +618,18 @@ void TestTScrubberBe::fake_a_scrub_set( ScrubGenerator::add_object(smap, obj, osd_num); } + shard_id_t shard = std::find_if(acting_shards.begin(), acting_shards.end(), + [&osd_num](pg_shard_t pg_shard) { + return osd_num == pg_shard.osd; + }) + ->shard; + std::cout << fmt::format("{}: {} inserting smap {:D}", __func__, osd_num, smap) << std::endl; - sbe->insert_faked_smap(pg_shard_t{osd_num}, smap); + sbe->insert_faked_smap(pg_shard_t{osd_num, shard}, smap); } // create the snap_mapper state @@ -574,7 +669,8 @@ class TestTScrubberBe_data_1 : public TestTScrubberBe { TestTScrubberBe_data_1() : TestTScrubberBe() {} // test configuration - pool_conf_t pl{3, 3, 3, 3, "rep_pool"}; + pool_conf_t pl{3, 3, 3, 3, "rep_pool", pg_pool_t::TYPE_REPLICATED, + std::nullopt}; TestTScrubberBeParams inject_params() override { @@ -673,7 +769,8 @@ class TestTScrubberBe_data_2 : public TestTScrubberBe { TestTScrubberBe_data_2() : TestTScrubberBe() {} // basic test configuration - 3 OSDs, all involved in the pool - pool_conf_t pl{3, 3, 3, 3, "rep_pool"}; + pool_conf_t pl{3, 3, 3, 3, "rep_pool", pg_pool_t::TYPE_REPLICATED, + std::nullopt}; TestTScrubberBeParams inject_params() override { -- 2.39.5