RealObjsConfList ScrubGenerator::make_real_objs_conf(
int64_t pool_id,
const RealObjsConf& blueprint,
- std::vector<int32_t> active_osds)
+ std::vector<int32_t> active_osds,
+ std::set<pg_shard_t> acting_shards,
+ bool erasure_coded_pool)
{
RealObjsConfList all_osds;
for (auto osd : active_osds) {
+ shard_id_t shard = std::find_if(acting_shards.begin(), acting_shards.end(),
+ [&osd](pg_shard_t pg_shard) {
+ return osd == pg_shard.osd;
+ })
+ ->shard;
+
RealObjsConfRef this_osd_fakes = std::make_unique<RealObjsConf>(blueprint);
// now - fix & corrupt every "object" in the blueprint
for (RealObj& robj : this_osd_fakes->objs) {
+ if (!erasure_coded_pool || robj.ghobj.shard_id == shard) {
+ robj.ghobj.hobj.pool = pool_id;
+ }
+ }
- robj.ghobj.hobj.pool = pool_id;
+ if (erasure_coded_pool) {
+ this_osd_fakes->objs.erase(std::remove_if(this_osd_fakes->objs.begin(),
+ this_osd_fakes->objs.end(),
+ [pool_id](RealObj& robj) {
+ return robj.ghobj.hobj.pool !=
+ pool_id;
+ }),
+ this_osd_fakes->objs.end());
}
all_osds[osd] = std::move(this_osd_fakes);
}
ret.smobj.size = blueprint.data.size;
ret.smobj.digest = blueprint.data.hash;
+ ret.smobj.digest_present = true;
/// \todo handle the 'present' etc'
ret.smobj.object_omap_keys = blueprint.data.omap.size();
// mocking the PG
class TestPg : public PgScrubBeListener {
+ ErasureCodeInterfaceRef m_erasure_code_interface;
+
public:
~TestPg() = default;
}
bool ec_can_decode(const shard_id_set& available_shards) const final {
- return false;
+ return get_is_ec_optimized() &&
+ available_shards.size() > get_ec_sinfo().get_k_plus_m() - 2;
};
+ // Fake encode function for erasure code tests in this class.
+ // Just sets parities to sum of data shards.
shard_id_map<bufferlist> ec_encode_acting_set(
- const bufferlist& in_bl) const final {
- return shard_id_map<bufferlist>(0);
+ const bufferlist& chunks) const final {
+ shard_id_map<bufferlist> encode_map(get_ec_sinfo().get_k_plus_m());
+ for (shard_id_t i; i < get_ec_sinfo().get_k_plus_m(); ++i) {
+ bufferlist bl;
+ bl.append(buffer::create(get_ec_sinfo().get_chunk_size(), 0));
+ bl.rebuild();
+ encode_map.insert(i, bl);
+ }
+ for (shard_id_t i; i < get_ec_sinfo().get_k(); ++i) {
+ for (int j = 0; j < get_ec_sinfo().get_chunk_size(); j++) {
+ encode_map.at(i).c_str()[j] =
+ chunks[j + (get_ec_sinfo().get_chunk_size() * i.id)];
+ for (shard_id_t k{static_cast<int8_t>(get_ec_sinfo().get_k_plus_m())};
+ k < get_ec_sinfo().get_k_plus_m(); ++k) {
+ encode_map.at(k).c_str()[j] +=
+ chunks[j + (get_ec_sinfo().get_chunk_size() * i.id)];
+ }
+ }
+ }
+
+ return encode_map;
};
+ // Fake encode function for erasure code tests in this class.
+ // Just sets calculates missing parity by using sum of all data shards =
+ // parity Tests using this will only have 1 missing shard.
shard_id_map<bufferlist> ec_decode_acting_set(
const shard_id_map<bufferlist>& chunks, int chunk_size) const final {
- return shard_id_map<bufferlist>(0);
- }
+ shard_id_map<bufferlist> decode_map(get_ec_sinfo().get_k_plus_m());
+
+ ceph_assert(chunks.size() > get_ec_sinfo().get_k_plus_m() - 2);
+
+ for (shard_id_t i; i < get_ec_sinfo().get_k_plus_m(); ++i) {
+ bufferlist bl;
+ bufferptr ptr = buffer::create(chunk_size, 0);
+ if (chunks.contains(i)) {
+ ptr.copy_in(0, chunk_size, chunks.at(i).to_str().c_str());
+ }
+ bl.append(ptr);
+ bl.rebuild();
+ decode_map.insert(i, bl);
+ }
+
+ for (shard_id_t shard; shard < get_ec_sinfo().get_k_plus_m(); ++shard) {
+ if (!chunks.contains(shard)) {
+ for (int j = 0; j < chunk_size; j++) {
+ if (shard < get_ec_sinfo().get_k()) {
+ decode_map.at(shard).c_str()[j] =
+ decode_map
+ .at(shard_id_t{static_cast<int8_t>(get_ec_sinfo().get_k())})
+ .c_str()[j];
+ }
+ for (shard_id_t i; i < get_ec_sinfo().get_k(); ++i) {
+ if (shard < get_ec_sinfo().get_k() && chunks.contains(i)) {
+ decode_map.at(shard).c_str()[j] -= decode_map.at(i).c_str()[j];
+ } else if (chunks.contains(i)) {
+ decode_map.at(shard).c_str()[j] += decode_map.at(i).c_str()[j];
+ }
+ }
+ }
+ } else {
+ decode_map.insert(shard, chunks.at(shard));
+ }
+ }
+
+ return decode_map;
+ };
bool get_ec_supports_crc_encode_decode() const final {
return get_is_ec_optimized();
* generated by the Primary). Then - create the snap-sets for all
* the objects in the set.
*/
- void fake_a_scrub_set(ScrubGenerator::RealObjsConfList& all_sets);
+ void fake_a_scrub_set(ScrubGenerator::RealObjsConfList& all_sets,
+ std::set<pg_shard_t> acting_shards);
std::unique_ptr<TestScrubBackend> sbe;
std::cout << fmt::format("PG info: {}", info) << std::endl;
real_objs_list =
- ScrubGenerator::make_real_objs_conf(pool_id, real_objs, acting_osds);
+ ScrubGenerator::make_real_objs_conf(pool_id,
+ real_objs,
+ acting_osds,
+ acting_shards,
+ pool_conf
+ .erasure_code_profile.has_value());
// now we can create the main mockers
}
sbe->new_chunk();
- fake_a_scrub_set(real_objs_list);
+ fake_a_scrub_set(real_objs_list, acting_shards);
}
}
osdmap->apply_incremental(pending_inc);
- // create a replicated pool
+ // create a pool
OSDMap::Incremental new_pool_inc(osdmap->get_epoch() + 1);
new_pool_inc.new_pool_max = osdmap->get_pool_max();
new_pool_inc.fsid = osdmap->get_fsid();
pg_pool_t empty;
auto p = new_pool_inc.get_new_pool(pool_id, &empty);
p->size = pconf.size;
+ p->min_size = pconf.min_size;
p->set_pg_num(pconf.pg_num);
p->set_pgp_num(pconf.pgp_num);
- p->type = pg_pool_t::TYPE_REPLICATED;
+ p->type = pconf.type;
p->crush_rule = 0;
p->set_flag(pg_pool_t::FLAG_HASHPSPOOL);
new_pool_inc.new_pool_names[pool_id] = pconf.name;
+ if (pconf.erasure_code_profile) {
+ osdmap->set_erasure_code_profile(
+ pconf.erasure_code_profile->erasure_code_profile_name,
+ pconf.erasure_code_profile->erasure_code_profile);
+ p->erasure_code_profile =
+ pconf.erasure_code_profile->erasure_code_profile_name;
+ p->set_flag(pg_pool_t::FLAG_EC_OVERWRITES);
+ p->set_flag(pg_pool_t::FLAG_EC_OPTIMIZATIONS);
+ }
osdmap->apply_incremental(new_pool_inc);
return osdmap;
}
<< std::endl;
spg = spg_t{pgid};
- i_am = pg_shard_t{up_primary};
- std::cout << fmt::format("{}: spg: {} and I am {}", __func__, spg, i_am)
- << std::endl;
// the 'acting shards' set - the one actually used by the scrubber
+ shard_id_t shard;
std::for_each(acting_osds.begin(), acting_osds.end(), [&](int osd) {
- acting_shards.insert(pg_shard_t{osd});
+ acting_shards.insert(pg_shard_t{osd, shard});
+ ++shard;
});
std::cout << fmt::format("{}: acting_shards: {}", __func__, acting_shards)
<< std::endl;
+ shard_id_t osd_shard =
+ std::find_if(
+ acting_shards.begin(), acting_shards.end(),
+ [&](pg_shard_t pg_shard) { return i_am.osd == pg_shard.osd; })
+ ->shard;
+
+ i_am = pg_shard_t{up_primary, osd_shard};
+ std::cout << fmt::format("{}: spg: {} and I am {}", __func__, spg, i_am)
+ << std::endl;
+
pg_info_t info;
info.pgid = spg;
/// \todo: handle the epochs:
}
void TestTScrubberBe::fake_a_scrub_set(
- ScrubGenerator::RealObjsConfList& all_sets)
-{
+ ScrubGenerator::RealObjsConfList& all_sets,
+ std::set<pg_shard_t> acting_shards) {
for (int osd_num = 0; osd_num < pool_conf.size; ++osd_num) {
ScrubMap smap;
smap.valid_through = eversion_t{1, 1};
ScrubGenerator::add_object(smap, obj, osd_num);
}
+ shard_id_t shard = std::find_if(acting_shards.begin(), acting_shards.end(),
+ [&osd_num](pg_shard_t pg_shard) {
+ return osd_num == pg_shard.osd;
+ })
+ ->shard;
+
std::cout << fmt::format("{}: {} inserting smap {:D}",
__func__,
osd_num,
smap)
<< std::endl;
- sbe->insert_faked_smap(pg_shard_t{osd_num}, smap);
+ sbe->insert_faked_smap(pg_shard_t{osd_num, shard}, smap);
}
// create the snap_mapper state
TestTScrubberBe_data_1() : TestTScrubberBe() {}
// test configuration
- pool_conf_t pl{3, 3, 3, 3, "rep_pool"};
+ pool_conf_t pl{3, 3, 3, 3, "rep_pool", pg_pool_t::TYPE_REPLICATED,
+ std::nullopt};
TestTScrubberBeParams inject_params() override
{
TestTScrubberBe_data_2() : TestTScrubberBe() {}
// basic test configuration - 3 OSDs, all involved in the pool
- pool_conf_t pl{3, 3, 3, 3, "rep_pool"};
+ pool_conf_t pl{3, 3, 3, 3, "rep_pool", pg_pool_t::TYPE_REPLICATED,
+ std::nullopt};
TestTScrubberBeParams inject_params() override
{
projects / ceph.git / commitdiff