From: Ronen Friedman Date: Wed, 20 Apr 2022 05:49:46 +0000 (+0000) Subject: test/osd/scrub: scrubber backend test files X-Git-Tag: v18.0.0~887^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b7d2b25232c3c50f012857d60c91728069e047b5;p=ceph.git test/osd/scrub: scrubber backend test files introducing the scrubber_generators to create scrubber test data, and the scrubber_test_datasets for pre-prepared test configurations. Signed-off-by: Ronen Friedman --- diff --git a/src/osd/osd_types_fmt.h b/src/osd/osd_types_fmt.h index f210425c6d15..58a634f70ca7 100644 --- a/src/osd/osd_types_fmt.h +++ b/src/osd/osd_types_fmt.h @@ -6,8 +6,9 @@ */ #include "common/hobject_fmt.h" -#include "osd/osd_types.h" #include "include/types_fmt.h" +#include "osd/osd_types.h" +#include template <> struct fmt::formatter { @@ -132,3 +133,212 @@ struct fmt::formatter { } } }; + +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template + auto format(const pg_history_t& pgh, FormatContext& ctx) + { + fmt::format_to(ctx.out(), + "ec={}/{} lis/c={}/{} les/c/f={}/{}/{} sis={}", + pgh.epoch_created, + pgh.epoch_pool_created, + pgh.last_interval_started, + pgh.last_interval_clean, + pgh.last_epoch_started, + pgh.last_epoch_clean, + pgh.last_epoch_marked_full, + pgh.same_interval_since); + + if (pgh.prior_readable_until_ub != ceph::timespan::zero()) { + return fmt::format_to(ctx.out(), + " pruub={}", + pgh.prior_readable_until_ub); + } else { + return ctx.out(); + } + } +}; + +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template + auto format(const pg_info_t& pgi, FormatContext& ctx) + { + fmt::format_to(ctx.out(), "{}({}", pgi.pgid, (pgi.dne() ? " DNE" : "")); + if (pgi.is_empty()) { + fmt::format_to(ctx.out(), " empty"); + } else { + fmt::format_to(ctx.out(), " v {}", pgi.last_update); + if (pgi.last_complete != pgi.last_update) { + fmt::format_to(ctx.out(), " lc {}", pgi.last_complete); + } + fmt::format_to(ctx.out(), " ({},{}]", pgi.log_tail, pgi.last_update); + } + if (pgi.is_incomplete()) { + fmt::format_to(ctx.out(), " lb {}", pgi.last_backfill); + } + fmt::format_to(ctx.out(), + " local-lis/les={}/{}", + pgi.last_interval_started, + pgi.last_epoch_started); + return fmt::format_to(ctx.out(), + " n={} {})", + pgi.stats.stats.sum.num_objects, + pgi.history); + } +}; + +// snaps and snap-sets + +template class C> +struct fmt::formatter> { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template + auto format(const interval_set& inter, FormatContext& ctx) + { + bool first = true; + fmt::format_to(ctx.out(), "["); + for (const auto& [start, len] : inter) { + fmt::format_to(ctx.out(), "{}{}~{}", (first ? "" : ","), start, len); + first = false; + } + return fmt::format_to(ctx.out(), "]"); + } +}; + +template <> +struct fmt::formatter { + template + constexpr auto parse(ParseContext& ctx) + { + auto it = ctx.begin(); + if (it != ctx.end() && *it == 'D') { + verbose = true; + ++it; + } + return it; + } + + template + auto format(const SnapSet& snps, FormatContext& ctx) + { + if (verbose) { + // similar to SnapSet::dump() + fmt::format_to(ctx.out(), + "snaps{{{}: clns ({}): ", + snps.seq, + snps.clones.size()); + for (auto cln : snps.clones) { + + fmt::format_to(ctx.out(), "[{}: sz:", cln); + + auto cs = snps.clone_size.find(cln); + if (cs != snps.clone_size.end()) { + fmt::format_to(ctx.out(), "{} ", cs->second); + } else { + fmt::format_to(ctx.out(), "??"); + } + + auto co = snps.clone_overlap.find(cln); + if (co != snps.clone_overlap.end()) { + fmt::format_to(ctx.out(), "olp:{} ", co->second); + } else { + fmt::format_to(ctx.out(), "olp:?? "); + } + + auto cln_snps = snps.clone_snaps.find(cln); + if (cln_snps != snps.clone_snaps.end()) { + fmt::format_to(ctx.out(), "cl-snps:{} ]", cln_snps->second); + } else { + fmt::format_to(ctx.out(), "cl-snps:?? ]"); + } + } + + return fmt::format_to(ctx.out(), "}}"); + + } else { + return fmt::format_to(ctx.out(), + "{}={}:{}", + snps.seq, + snps.snaps, + snps.clone_snaps); + } + } + + bool verbose{false}; +}; + +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + ///\todo: consider passing the 'D" flag to control snapset dump + template + auto format(const ScrubMap::object& so, FormatContext& ctx) + { + fmt::format_to(ctx.out(), + "so{{ sz:{} dd:{} od:{} ", + so.size, + so.digest, + so.digest_present); + + // note the special handling of (1) OI_ATTR and (2) non-printables + for (auto [k, v] : so.attrs) { + std::string bkstr{v.raw_c_str(), v.raw_length()}; + if (k == std::string{OI_ATTR}) { + /// \todo consider parsing the OI args here. Maybe add a specific format + /// specifier + fmt::format_to(ctx.out(), "{{{}:<>({})}} ", k, bkstr.length()); + } else if (k == std::string{SS_ATTR}) { + bufferlist bl; + bl.push_back(v); + SnapSet sns{bl}; + fmt::format_to(ctx.out(), "{{{}:{:D}}} ", k, sns); + } else { + fmt::format_to(ctx.out(), "{{{}:{}({})}} ", k, bkstr, bkstr.length()); + } + } + + return fmt::format_to(ctx.out(), "}}"); + } +}; + +template <> +struct fmt::formatter { + template + constexpr auto parse(ParseContext& ctx) + { + auto it = ctx.begin(); + if (it != ctx.end() && *it == 'D') { + debug_log = true; // list the objects + ++it; + } + return it; + } + + template + auto format(const ScrubMap& smap, FormatContext& ctx) + { + fmt::format_to(ctx.out(), + "smap{{ valid:{} incr-since:{} #:{}", + smap.valid_through, + smap.incr_since, + smap.objects.size()); + if (debug_log) { + fmt::format_to(ctx.out(), " objects:"); + for (const auto& [ho, so] : smap.objects) { + fmt::format_to(ctx.out(), "\n\th.o<{}>:<{}> ", ho, so); + } + fmt::format_to(ctx.out(), "\n"); + } + return fmt::format_to(ctx.out(), "}}"); + } + + bool debug_log{false}; +}; diff --git a/src/test/osd/CMakeLists.txt b/src/test/osd/CMakeLists.txt index b2c5ec884da0..863f7d6e9665 100644 --- a/src/test/osd/CMakeLists.txt +++ b/src/test/osd/CMakeLists.txt @@ -67,6 +67,15 @@ add_executable(unittest_osdscrub add_ceph_unittest(unittest_osdscrub) target_link_libraries(unittest_osdscrub osd os global ${CMAKE_DL_LIBS} mon ${BLKID_LIBRARIES}) +# unittest_scrubber_be +add_executable(unittest_scrubber_be + test_scrubber_be.cc + scrubber_generators.cc + scrubber_test_datasets.cc + ) +add_ceph_unittest(unittest_scrubber_be) +target_link_libraries(unittest_scrubber_be osd os global ${CMAKE_DL_LIBS} mon ${BLKID_LIBRARIES}) + # unittest_pglog add_executable(unittest_pglog TestPGLog.cc diff --git a/src/test/osd/scrubber_generators.cc b/src/test/osd/scrubber_generators.cc new file mode 100644 index 000000000000..a8a686d30f13 --- /dev/null +++ b/src/test/osd/scrubber_generators.cc @@ -0,0 +1,165 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/osd/scrubber_generators.h" + +using namespace ScrubGenerator; + +// ref: PGLogTestRebuildMissing() +bufferptr create_object_info(const ScrubGenerator::RealObj& objver) +{ + object_info_t oi{}; + oi.soid = objver.ghobj.hobj; + oi.version = eversion_t(objver.ghobj.generation, 0); + oi.size = objver.data.size; + + bufferlist bl; + oi.encode(bl, + 0 /*get_osdmap()->get_features(CEPH_ENTITY_TYPE_OSD, nullptr)*/); + bufferptr bp(bl.c_str(), bl.length()); + return bp; +} + +std::pair> create_object_snapset( + const ScrubGenerator::RealObj& robj, + const SnapsetMockData* snapset_mock_data) +{ + if (!snapset_mock_data) { + return {bufferptr(), {}}; + } + /// \todo fill in missing version/osd details from the robj + auto sns = snapset_mock_data->make_snapset(); + bufferlist bl; + encode(sns, bl); + bufferptr bp = bufferptr(bl.c_str(), bl.length()); + + // extract the set of object snaps + return {bp, sns.snaps}; +} + +RealObjsConfList ScrubGenerator::make_real_objs_conf( + int64_t pool_id, + const RealObjsConf& blueprint, + std::vector active_osds) +{ + RealObjsConfList all_osds; + + for (auto osd : active_osds) { + RealObjsConfRef this_osd_fakes = std::make_unique(blueprint); + // now - fix & corrupt every "object" in the blueprint + for (RealObj& robj : this_osd_fakes->objs) { + + robj.ghobj.hobj.pool = pool_id; + } + + all_osds[osd] = std::move(this_osd_fakes); + } + return all_osds; // reconsider (maybe add a move ctor?) +} + +///\todo dispose of the created buffer pointers + +ScrubGenerator::SmapEntry ScrubGenerator::make_smobject( + const ScrubGenerator::RealObj& blueprint, + int osd_num) +{ + ScrubGenerator::SmapEntry ret; + + ret.ghobj = blueprint.ghobj; + ret.smobj.attrs[OI_ATTR] = create_object_info(blueprint); + if (blueprint.snapset_mock_data) { + auto [bp, snaps] = + create_object_snapset(blueprint, blueprint.snapset_mock_data); + ret.smobj.attrs[SS_ATTR] = bp; + std::cout << fmt::format("{}: ({}) osd:{} snaps:{}", + __func__, + ret.ghobj.hobj, + osd_num, + snaps) + << std::endl; + } + + for (const auto& [at_k, at_v] : blueprint.data.attrs) { + ret.smobj.attrs[at_k] = ceph::buffer::copy(at_v.c_str(), at_v.size()); + { + // verifying (to be removed after dev phase) + auto bk = ret.smobj.attrs[at_k].clone(); + std::string bkstr{bk.get()->get_data(), bk.get()->get_len()}; + std::cout << fmt::format("{}: verification: {}", __func__, bkstr) + << std::endl; + } + } + ret.smobj.size = blueprint.data.size; + ret.smobj.digest = blueprint.data.hash; + /// \todo handle the 'present' etc' + + ret.smobj.object_omap_keys = blueprint.data.omap.size(); + ret.smobj.object_omap_bytes = blueprint.data.omap_bytes; + return ret; +} + +all_clones_snaps_t ScrubGenerator::all_clones( + const ScrubGenerator::RealObj& head_obj) +{ + std::cout << fmt::format("{}: head_obj.ghobj.hobj:{}", + __func__, + head_obj.ghobj.hobj) + << std::endl; + + std::map> ret; + + for (const auto& clone : head_obj.snapset_mock_data->clones) { + auto clone_set_it = head_obj.snapset_mock_data->clone_snaps.find(clone); + if (clone_set_it == head_obj.snapset_mock_data->clone_snaps.end()) { + std::cout << "note: no clone_snaps for " << clone << std::endl; + continue; + } + auto clone_set = clone_set_it->second; + hobject_t clone_hobj{head_obj.ghobj.hobj}; + clone_hobj.snap = clone; + + ret[clone_hobj] = clone_set_it->second; + std::cout << fmt::format("{}: clone:{} clone_set:{}", + __func__, + clone_hobj, + clone_set) + << std::endl; + } + + return ret; +} + +void ScrubGenerator::add_object(ScrubMap& map, + const ScrubGenerator::RealObj& real_obj, + int osd_num) +{ + // do we have data corruption recipe for this OSD? + /// \todo c++20: use contains() + CorruptFunc relevant_fix = crpt_do_nothing; + + auto p = real_obj.corrupt_funcs->find(osd_num); + if (p != real_obj.corrupt_funcs->end()) { + // yes, we have a corruption recepie for this OSD + // \todo c++20: use at() + relevant_fix = p->second; + } + + // create a possibly-corrupted copy of the "real object" + auto modified_obj = (relevant_fix)(real_obj, osd_num); + + std::cout << fmt::format("{}: modified: osd:{} ho:{} key:{}", + __func__, + osd_num, + modified_obj.ghobj.hobj, + modified_obj.ghobj.hobj.get_key()) + << std::endl; + + auto entry = make_smobject(modified_obj, osd_num); + std::cout << fmt::format("{}: osd:{} smap entry: {} {}", + __func__, + osd_num, + entry.smobj.size, + entry.smobj.attrs.size()) + << std::endl; + map.objects[entry.ghobj.hobj] = entry.smobj; +} diff --git a/src/test/osd/scrubber_generators.h b/src/test/osd/scrubber_generators.h new file mode 100644 index 000000000000..d0cbb22c4c80 --- /dev/null +++ b/src/test/osd/scrubber_generators.h @@ -0,0 +1,266 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#pragma once + +/// \file generating scrub-related maps & objects for unit tests + +#include +#include +#include +#include +#include +#include + +#include "include/buffer.h" +#include "include/buffer_raw.h" +#include "include/object_fmt.h" +#include "osd/osd_types_fmt.h" +#include "osd/scrubber/pg_scrubber.h" + +namespace ScrubGenerator { + +/// \todo enhance the MockLog to capture the log messages +class MockLog : public LoggerSinkSet { + public: + void debug(std::stringstream& s) final + { + std::cout << "\n<> " << s.str() << std::endl; + } + void info(std::stringstream& s) final + { + std::cout << "\n<> " << s.str() << std::endl; + } + void sec(std::stringstream& s) final + { + std::cout << "\n<> " << s.str() << std::endl; + } + void warn(std::stringstream& s) final + { + std::cout << "\n<> " << s.str() << std::endl; + } + void error(std::stringstream& s) final + { + err_count++; + std::cout << "\n<> " << s.str() << std::endl; + } + OstreamTemp info() final { return OstreamTemp(CLOG_INFO, this); } + OstreamTemp warn() final { return OstreamTemp(CLOG_WARN, this); } + OstreamTemp error() final { return OstreamTemp(CLOG_ERROR, this); } + OstreamTemp sec() final { return OstreamTemp(CLOG_ERROR, this); } + OstreamTemp debug() final { return OstreamTemp(CLOG_DEBUG, this); } + + void do_log(clog_type prio, std::stringstream& ss) final + { + switch (prio) { + case CLOG_DEBUG: + debug(ss); + break; + case CLOG_INFO: + info(ss); + break; + case CLOG_SEC: + sec(ss); + break; + case CLOG_WARN: + warn(ss); + break; + case CLOG_ERROR: + default: + error(ss); + break; + } + } + + void do_log(clog_type prio, const std::string& ss) final + { + switch (prio) { + case CLOG_DEBUG: + debug() << ss; + break; + case CLOG_INFO: + info() << ss; + break; + case CLOG_SEC: + sec() << ss; + break; + case CLOG_WARN: + warn() << ss; + break; + case CLOG_ERROR: + default: + error() << ss; + break; + } + } + + virtual ~MockLog() {} + + int err_count{0}; + int expected_err_count{0}; + void set_expected_err_count(int c) { expected_err_count = c; } +}; + +// ///////////////////////////////////////////////////////////////////////// // +// ///////////////////////////////////////////////////////////////////////// // + +struct pool_conf_t { + int pg_num{3}; + int pgp_num{3}; + int size{3}; + int min_size{3}; + std::string name{"rep_pool"}; +}; + +using attr_t = std::map; + +using all_clones_snaps_t = std::map>; + +struct RealObj; + +// a function to manipulate (i.e. corrupt) an object in a specific OSD +using CorruptFunc = + std::function; +using CorruptFuncList = std::map; // per OSD + +struct SnapsetMockData { + + using CookedCloneSnaps = + std::tuple, + std::map>, + std::map>>; + + // an auxiliary function to cook the data for the SnapsetMockData + using clone_snaps_cooker = CookedCloneSnaps (*)(); + + snapid_t seq; + std::vector snaps; // descending + std::vector clones; // ascending + + std::map> clone_overlap; // overlap w/ next + // newest + std::map clone_size; + std::map> clone_snaps; // descending + + + SnapsetMockData(snapid_t seq, + std::vector snaps, + std::vector clones, + std::map> clone_overlap, + std::map clone_size, + std::map> clone_snaps) + : seq(seq) + , snaps(snaps) + , clones(clones) + , clone_overlap(clone_overlap) + , clone_size(clone_size) + , clone_snaps(clone_snaps) + {} + + SnapsetMockData(snapid_t seq, + std::vector snaps, + std::vector clones, + clone_snaps_cooker func) + : seq{seq} + , snaps{snaps} + , clones(clones) + { + auto [clone_size_, clone_snaps_, clone_overlap_] = func(); + clone_size = clone_size_; + clone_snaps = clone_snaps_; + clone_overlap = clone_overlap_; + } + + SnapSet make_snapset() const + { + SnapSet ss; + ss.seq = seq; + ss.snaps = snaps; + ss.clones = clones; + ss.clone_overlap = clone_overlap; + ss.clone_size = clone_size; + ss.clone_snaps = clone_snaps; + return ss; + } +}; + +// an object in our "DB" - with its versioned snaps, "data" (size and hash), +// and "omap" (size and hash) + +struct RealData { + // not needed at this level of "data falsification": std::byte data; + uint64_t size; + uint32_t hash; + uint32_t omap_digest; + uint32_t omap_bytes; + attr_t omap; + attr_t attrs; +}; + +struct RealObj { + // the ghobject - oid, version, snap, hash, pool + ghobject_t ghobj; + RealData data; + const CorruptFuncList* corrupt_funcs; + const SnapsetMockData* snapset_mock_data; +}; + +static inline RealObj crpt_do_nothing(const RealObj& s, int osdn) +{ + return s; +} + +struct SmapEntry { + ghobject_t ghobj; + ScrubMap::object smobj; +}; + + +ScrubGenerator::SmapEntry make_smobject( + const ScrubGenerator::RealObj& blueprint, // the whole set of versions + int osd_num); + + +/** + * returns the object's snap-set + */ +void add_object(ScrubMap& map, const RealObj& obj_versions, int osd_num); + +struct RealObjsConf { + std::vector objs; +}; + +using RealObjsConfRef = std::unique_ptr; + +// RealObjsConf will be "developed" into the following of per-osd sets, +// now with the correct pool ID, and with the corrupting functions +// activated on the data +using RealObjsConfList = std::map; + +RealObjsConfList make_real_objs_conf(int64_t pool_id, + const RealObjsConf& blueprint, + std::vector active_osds); + +/** + * create the snap-ids set for all clones appearing in the head + * object's snapset (those will be injected into the scrubber's mock, + * to be used as the 'snap_mapper') + */ +all_clones_snaps_t all_clones(const RealObj& head_obj); +} // namespace ScrubGenerator + +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template + auto format(const ScrubGenerator::RealObj& rlo, FormatContext& ctx) + { + using namespace ScrubGenerator; + return fmt::format_to(ctx.out(), + "RealObj(gh:{}, dt:{}, snaps:{})", + rlo.ghobj, + rlo.data.size, + (rlo.snapset_mock_data ? rlo.snapset_mock_data->snaps + : std::vector{})); + } +}; diff --git a/src/test/osd/scrubber_test_datasets.cc b/src/test/osd/scrubber_test_datasets.cc new file mode 100644 index 000000000000..478fd25fe953 --- /dev/null +++ b/src/test/osd/scrubber_test_datasets.cc @@ -0,0 +1,120 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/// \file data-sets used by the scrubber unit tests + +#include "./scrubber_test_datasets.h" + + +using namespace ScrubGenerator; +using namespace std::string_literals; + +namespace ScrubDatasets { + +static RealObj corrupt_object_size(const RealObj& s, [[maybe_unused]] int osdn) +{ + RealObj ret = s; + ret.data.size = s.data.size + 1; + return ret; +} + +static RealObj corrupt_nothing(const RealObj& s, int osdn) +{ + return s; +} + + +static CorruptFuncList crpt_funcs_set0 = {{0, &corrupt_nothing}}; + +CorruptFuncList crpt_funcs_set1 = {{0, &corrupt_object_size}, + {1, &corrupt_nothing}}; + + +// object with head & two snaps + +static hobject_t hobj_ms1{object_t{"hobj_ms1"}, + "keykey", // key + CEPH_NOSNAP, // snap_id + 0, // hash + 0, // pool + ""s}; // nspace + +SnapsetMockData::CookedCloneSnaps ms1_fn() +{ + std::map clnsz; + clnsz[0x20] = 222; + clnsz[0x30] = 333; + + std::map> clnsn; + clnsn[0x20] = {0x20}; + clnsn[0x30] = {0x30}; + + std::map> overlaps; + overlaps[0x20] = {}; + overlaps[0x30] = {}; + return {clnsz, clnsn, overlaps}; +} + +static SnapsetMockData hobj_ms1_snapset{/* seq */ 0x40, + /* snaps */ {0x30, 0x20}, + /* clones */ {0x20, 0x30}, + ms1_fn}; + +hobject_t hobj_ms1_snp30{object_t{"hobj_ms1"}, + "keykey", // key + 0x30, // snap_id + 0, // hash + 0, // pool + ""s}; // nspace + +static hobject_t hobj_ms1_snp20{object_t{"hobj_ms1"}, + "keykey", // key + 0x20, // snap_id + 0, // hash + 0, // pool + ""s}; // nspace + + +ScrubGenerator::RealObjsConf minimal_snaps_configuration{ + /* RealObjsConf::objs */ { + + /* Clone 30 */ { + ghobject_t{hobj_ms1_snp30, 0, shard_id_t{0}}, + RealData{ + 333, + 0x17, + 17, + 21, + attr_t{/*{"_om1k", "om1v"}, {"om1k", "om1v"},*/ {"om3k", "om3v"}}, + attr_t{{"_at1k", "_at1v"}, {"_at2k", "at2v"}, {"at3k", "at3v"}}}, + &crpt_funcs_set0, + nullptr}, + + /* Clone 20 */ + {ghobject_t{hobj_ms1_snp20, 0, shard_id_t{0}}, + RealData{222, + 0x17, + 17, + 21, + attr_t{/*{"_om1k", "om1v"}, {"om1k", "om1v"},*/ {"om3k", "om3v"}}, + attr_t{{"_at1k", "_at1v"}, {"_at2k", "at2v"}, {"at3k", "at3v"}}}, + &crpt_funcs_set0, + nullptr}, + + /* Head */ + {ghobject_t{hobj_ms1, 0, shard_id_t{0}}, + RealData{100, + 0x17, + 17, + 21, + attr_t{{"_om1k", "om1v"}, {"om1k", "om1v"}, {"om3k", "om3v"}}, + attr_t{{"_at1k", "_at1v"}, {"_at2k", "at2v"}, {"at3k", "at3v"}} + + + }, + &crpt_funcs_set0, + &hobj_ms1_snapset}} + +}; + +} // namespace ScrubDatasets diff --git a/src/test/osd/scrubber_test_datasets.h b/src/test/osd/scrubber_test_datasets.h new file mode 100644 index 000000000000..181528568303 --- /dev/null +++ b/src/test/osd/scrubber_test_datasets.h @@ -0,0 +1,21 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#pragma once + +/// \file data-sets used by the scrubber unit tests + +#include "./scrubber_generators.h" + +namespace ScrubDatasets { +/* + * Two objects with some clones. No inconsitencies. + */ +extern ScrubGenerator::RealObjsConf minimal_snaps_configuration; + +// and a part of this configuration, one that we will corrupt in a test: +extern hobject_t hobj_ms1_snp30; + +// a manipulation set used in TestTScrubberBe_data_2: +extern ScrubGenerator::CorruptFuncList crpt_funcs_set1; + +} // namespace ScrubDatasets diff --git a/src/test/osd/test_scrubber_be.cc b/src/test/osd/test_scrubber_be.cc new file mode 100644 index 000000000000..d42634020af9 --- /dev/null +++ b/src/test/osd/test_scrubber_be.cc @@ -0,0 +1,642 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#include "./scrubber_generators.h" +#include "./scrubber_test_datasets.h" + +#include +#include +#include + +#include "common/async/context_pool.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "global/global_init.h" +#include "mon/MonClient.h" +#include "msg/Messenger.h" +#include "os/ObjectStore.h" +#include "osd/PG.h" +#include "osd/PGBackend.h" +#include "osd/PrimaryLogPG.h" +#include "osd/osd_types.h" +#include "osd/osd_types_fmt.h" +#include "osd/scrubber/pg_scrubber.h" +#include "osd/scrubber/scrub_backend.h" + +/// \file testing isolated parts of the Scrubber backend + +using namespace std::string_literals; + +int main(int argc, char** argv) +{ + std::map defaults = { + // make sure we have 3 copies, or some tests won't work + {"osd_pool_default_size", "3"}, + // our map is flat, so just try and split across OSDs, not hosts or whatever + {"osd_crush_chooseleaf_type", "0"}, + }; + std::vector args(argv, argv + argc); + auto cct = global_init(&defaults, + args, + CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + +class TestScrubBackend : public ScrubBackend { + public: + TestScrubBackend(ScrubBeListener& scrubber, + PgScrubBeListener& pg, + pg_shard_t i_am, + bool repair, + scrub_level_t shallow_or_deep, + const std::set& acting) + : ScrubBackend(scrubber, pg, i_am, repair, shallow_or_deep, acting) + {} + + bool get_m_repair() const { return m_repair; } + bool get_is_replicated() const { return m_is_replicated; } + auto get_omap_stats() const { return m_omap_stats; } + + const std::vector& all_but_me() const { return m_acting_but_me; } + + /// populate the scrub-maps set for the 'chunk' being scrubbed + void insert_faked_smap(pg_shard_t shard, const ScrubMap& smap); +}; + +// mocking the PG +class TestPg : public PgScrubBeListener { + public: + ~TestPg() = default; + + TestPg(std::shared_ptr pool, pg_info_t& pginfo, pg_shard_t my_osd) + : m_pool{pool} + , m_info{pginfo} + , m_pshard{my_osd} + {} + + const PGPool& get_pgpool() const final { return *(m_pool.get()); } + pg_shard_t get_primary() const final { return m_pshard; } + void force_object_missing(ScrubberPasskey, + const std::set& peer, + const hobject_t& oid, + eversion_t version) final + {} + + const pg_info_t& get_pg_info(ScrubberPasskey) const final { return m_info; } + + uint64_t logical_to_ondisk_size(uint64_t logical_size) const final + { + return logical_size; + } + + bool is_waiting_for_unreadable_object() const final { return false; } + + std::shared_ptr m_pool; + pg_info_t& m_info; + pg_shard_t m_pshard; +}; + + +// /////////////////////////////////////////////////////////////////////////// +// /////////////////////////////////////////////////////////////////////////// + +// and the scrubber +class TestScrubber : public ScrubBeListener, public SnapMapperAccessor { + public: + ~TestScrubber() = default; + + TestScrubber(spg_t spg, OSDMapRef osdmap, LoggerSinkSet& logger) + : m_spg{spg} + , m_logger{logger} + , m_osdmap{osdmap} + {} + + std::ostream& gen_prefix(std::ostream& out) const final { return out; } + + CephContext* get_pg_cct() const final { return g_ceph_context; } + + LoggerSinkSet& get_logger() const final { return m_logger; } + + bool is_primary() const final { return m_primary; } + + spg_t get_pgid() const final { return m_info.pgid; } + + const OSDMapRef& get_osdmap() const final { return m_osdmap; } + + void add_to_stats(const object_stat_sum_t& stat) final { m_stats.add(stat); } + + // submit_digest_fixes() mock can be set to expect a specific set of + // fixes to perform. + /// \todo implement the mock. + void submit_digest_fixes(const digests_fixes_t& fixes) final + { + std::cout << fmt::format("{} submit_digest_fixes({})", + __func__, + fmt::join(fixes, ",")) + << std::endl; + } + + int get_snaps(const hobject_t& hoid, + std::set* snaps_set) const final; + + void set_snaps(const hobject_t& hoid, const std::vector& snaps) + { + std::cout + << fmt::format("{}: ({}) -> #{} {}", __func__, hoid, snaps.size(), snaps) + << std::endl; + std::set snaps_set(snaps.begin(), snaps.end()); + m_snaps[hoid] = snaps_set; + } + + void set_snaps(const ScrubGenerator::all_clones_snaps_t& clones_snaps) + { + for (const auto& [clone, snaps] : clones_snaps) { + std::cout << fmt::format("{}: ({}) -> #{} {}", + __func__, + clone, + snaps.size(), + snaps) + << std::endl; + std::set snaps_set(snaps.begin(), snaps.end()); + m_snaps[clone] = snaps_set; + } + } + + bool m_primary{true}; + spg_t m_spg; + LoggerSinkSet& m_logger; + OSDMapRef m_osdmap; + pg_info_t m_info; + object_stat_sum_t m_stats; + + // the "snap-mapper" database (returned by get_snaps()) + std::map> m_snaps; +}; + +int TestScrubber::get_snaps(const hobject_t& hoid, + std::set* snaps_set) const +{ + auto it = m_snaps.find(hoid); + if (it == m_snaps.end()) { + std::cout << fmt::format("{}: ({}) no snaps", __func__, hoid) << std::endl; + return -ENOENT; + } + + *snaps_set = it->second; + std::cout << fmt::format("{}: ({}) -> #{} {}", + __func__, + hoid, + snaps_set->size(), + *snaps_set) + << std::endl; + return 0; +} + + +// /////////////////////////////////////////////////////////////////////////// +// /////////////////////////////////////////////////////////////////////////// + + +/// parameters for TestTScrubberBe construction +struct TestTScrubberBeParams { + ScrubGenerator::pool_conf_t pool_conf; + ScrubGenerator::RealObjsConf objs_conf; + int num_osds; +}; + + +// /////////////////////////////////////////////////////////////////////////// +// /////////////////////////////////////////////////////////////////////////// + + +// the actual owner of the OSD "objects" that are used by +// the mockers +class TestTScrubberBe : public ::testing::Test { + public: + // the test data source + virtual TestTScrubberBeParams inject_params() = 0; + + // initial test data + ScrubGenerator::MockLog logger; + ScrubGenerator::pool_conf_t pool_conf; + ScrubGenerator::RealObjsConf real_objs; + int num_osds{0}; + + // ctor & initialization + + TestTScrubberBe() = default; + ~TestTScrubberBe() = default; + void SetUp() override; + void TearDown() override; + + /** + * Create the set of scrub-maps supposedly sent by the replica (or + * generated by the Primary). Then - create the snap-sets for all + * the objects in the set. + */ + void fake_a_scrub_set(ScrubGenerator::RealObjsConfList& all_sets); + + std::unique_ptr sbe; + + spg_t spg; + pg_shard_t i_am; // set to 'my osd and no shard' + std::set acting_shards; + std::vector acting_osds; + int acting_primary; + + std::unique_ptr test_scrubber; + + int64_t pool_id; + pg_pool_t pool_info; + + OSDMapRef osdmap; + + std::shared_ptr pool; + pg_info_t info; + + std::unique_ptr test_pg; + + // generated sets of "objects" for the active OSDs + ScrubGenerator::RealObjsConfList real_objs_list; + + protected: + /** + * Create the OSDmap and populate it with one pool, based on + * the pool configuration. + * For now - only replicated pools are supported. + */ + OSDMapRef setup_map(int num_osds, const ScrubGenerator::pool_conf_t& pconf); + + /** + * Create a PG in the one pool we have. Fake the PG info. + * Use the primary of the PG to determine "who we are". + * + * \returns the PG info + */ + pg_info_t setup_pg_in_map(); +}; + + +// /////////////////////////////////////////////////////////////////////////// +// /////////////////////////////////////////////////////////////////////////// + +void TestTScrubberBe::SetUp() +{ + std::cout << "TestTScrubberBe::SetUp()" << std::endl; + logger.err_count = 0; + + // fetch test configuration + auto params = inject_params(); + pool_conf = params.pool_conf; + real_objs = params.objs_conf; + num_osds = params.num_osds; + + // create the OSDMap + + osdmap = setup_map(num_osds, pool_conf); + + std::cout << "osdmap: " << *osdmap << std::endl; + + // extract the pool from the osdmap + + pool_id = osdmap->lookup_pg_pool_name(pool_conf.name); + const pg_pool_t* ext_pool_info = osdmap->get_pg_pool(pool_id); + pool = + std::make_shared(osdmap, pool_id, *ext_pool_info, pool_conf.name); + + std::cout << "pool: " << pool->info << std::endl; + + // a PG in that pool? + info = setup_pg_in_map(); + std::cout << fmt::format("PG info: {}", info) << std::endl; + + real_objs_list = + ScrubGenerator::make_real_objs_conf(pool_id, real_objs, acting_osds); + + // now we can create the main mockers + + // the "PgScrubber" + test_scrubber = std::make_unique(spg, osdmap, logger); + + // the "PG" (and its backend) + test_pg = std::make_unique(pool, info, i_am); + std::cout << fmt::format("{}: acting: {}", __func__, acting_shards) + << std::endl; + sbe = std::make_unique(*test_scrubber, + *test_pg, + i_am, + /* repair? */ false, + scrub_level_t::deep, + acting_shards); + + // create a osd-num only copy of the relevant OSDs + acting_osds.reserve(acting_shards.size()); + for (const auto& shard : acting_shards) { + acting_osds.push_back(shard.osd); + } + + sbe->new_chunk(); + fake_a_scrub_set(real_objs_list); +} + + +// Note: based on TestOSDMap.cc. +OSDMapRef TestTScrubberBe::setup_map(int num_osds, + const ScrubGenerator::pool_conf_t& pconf) +{ + auto osdmap = std::make_shared(); + uuid_d fsid; + osdmap->build_simple(g_ceph_context, 0, fsid, num_osds); + OSDMap::Incremental pending_inc(osdmap->get_epoch() + 1); + pending_inc.fsid = osdmap->get_fsid(); + entity_addrvec_t sample_addrs; + sample_addrs.v.push_back(entity_addr_t()); + uuid_d sample_uuid; + for (int i = 0; i < num_osds; ++i) { + sample_uuid.generate_random(); + sample_addrs.v[0].nonce = i; + pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW; + pending_inc.new_up_client[i] = sample_addrs; + pending_inc.new_up_cluster[i] = sample_addrs; + pending_inc.new_hb_back_up[i] = sample_addrs; + pending_inc.new_hb_front_up[i] = sample_addrs; + pending_inc.new_weight[i] = CEPH_OSD_IN; + pending_inc.new_uuid[i] = sample_uuid; + } + osdmap->apply_incremental(pending_inc); + + // create a replicated pool + OSDMap::Incremental new_pool_inc(osdmap->get_epoch() + 1); + new_pool_inc.new_pool_max = osdmap->get_pool_max(); + new_pool_inc.fsid = osdmap->get_fsid(); + uint64_t pool_id = ++new_pool_inc.new_pool_max; + pg_pool_t empty; + auto p = new_pool_inc.get_new_pool(pool_id, &empty); + p->size = pconf.size; + p->set_pg_num(pconf.pg_num); + p->set_pgp_num(pconf.pgp_num); + p->type = pg_pool_t::TYPE_REPLICATED; + p->crush_rule = 0; + p->set_flag(pg_pool_t::FLAG_HASHPSPOOL); + new_pool_inc.new_pool_names[pool_id] = pconf.name; + osdmap->apply_incremental(new_pool_inc); + return osdmap; +} + +pg_info_t TestTScrubberBe::setup_pg_in_map() +{ + pg_t rawpg(0, pool_id); + pg_t pgid = osdmap->raw_pg_to_pg(rawpg); + std::vector up_osds; + int up_primary; + + osdmap->pg_to_up_acting_osds(pgid, + &up_osds, + &up_primary, + &acting_osds, + &acting_primary); + + std::cout << fmt::format( + "{}: pg: {} up_osds: {} up_primary: {} acting_osds: {} " + "acting_primary: " + "{}", + __func__, + pgid, + up_osds, + up_primary, + acting_osds, + acting_primary) + << std::endl; + + spg = spg_t{pgid}; + i_am = pg_shard_t{up_primary}; + std::cout << fmt::format("{}: spg: {} and I am {}", __func__, spg, i_am) + << std::endl; + + // the 'acting shards' set - the one actually used by the scrubber + std::for_each(acting_osds.begin(), acting_osds.end(), [&](int osd) { + acting_shards.insert(pg_shard_t{osd}); + }); + std::cout << fmt::format("{}: acting_shards: {}", __func__, acting_shards) + << std::endl; + + pg_info_t info; + info.pgid = spg; + /// \todo: handle the epochs: + // info.last_update = osdmap->get_epoch(); + // info.last_complete = osdmap->get_epoch(); + // info.last_osdmap_epoch = osdmap->get_epoch(); + // info.history.last_epoch_marked_removed = osdmap->get_epoch(); + info.last_user_version = 1; + info.purged_snaps = {}; + info.last_user_version = 1; + info.history.last_epoch_clean = osdmap->get_epoch(); + info.history.last_epoch_split = osdmap->get_epoch(); + info.history.last_epoch_marked_full = osdmap->get_epoch(); + info.last_backfill = hobject_t::get_max(); + return info; +} + +void TestTScrubberBe::TearDown() +{ + EXPECT_EQ(logger.err_count, logger.expected_err_count); +} + +void TestTScrubberBe::fake_a_scrub_set( + ScrubGenerator::RealObjsConfList& all_sets) +{ + for (int osd_num = 0; osd_num < pool_conf.size; ++osd_num) { + ScrubMap smap; + smap.valid_through = eversion_t{1, 1}; + smap.incr_since = eversion_t{1, 1}; + smap.has_omap_keys = true; // to force omap checks + + // fill the map with the objects relevant to this OSD + for (auto& obj : all_sets[osd_num]->objs) { + std::cout << fmt::format("{}: object: {}", __func__, obj.ghobj.hobj) + << std::endl; + ScrubGenerator::add_object(smap, obj, osd_num); + } + + std::cout << fmt::format("{}: {} inserting smap {:D}", + __func__, + osd_num, + smap) + << std::endl; + sbe->insert_faked_smap(pg_shard_t{osd_num}, smap); + } + + // create the snap_mapper state + + for (const auto& robj : all_sets[i_am.osd]->objs) { + + std::cout << fmt::format("{}: object: {}", __func__, robj.ghobj.hobj) + << std::endl; + + if (robj.ghobj.hobj.snap == CEPH_NOSNAP) { + // head object + auto objects_snapset = ScrubGenerator::all_clones(robj); + test_scrubber->set_snaps(objects_snapset); + } + } +} + +void TestScrubBackend::insert_faked_smap(pg_shard_t shard, const ScrubMap& smap) +{ + ASSERT_TRUE(this_chunk.has_value()); + std::cout << fmt::format("{}: inserting faked smap for osd {}", + __func__, + shard.osd) + << std::endl; + this_chunk->received_maps[shard] = smap; +} + + +// /////////////////////////////////////////////////////////////////////////// +// /////////////////////////////////////////////////////////////////////////// + + +using namespace ScrubGenerator; + +class TestTScrubberBe_data_1 : public TestTScrubberBe { + public: + TestTScrubberBe_data_1() : TestTScrubberBe() {} + + // test configuration + pool_conf_t pl{3, 3, 3, 3, "rep_pool"}; + + TestTScrubberBeParams inject_params() override + { + std::cout << fmt::format("{}: injecting params (minimal snaps conf.)", + __func__) + << std::endl; + return TestTScrubberBeParams{ + /* pool_conf */ pl, + /* real_objs_conf */ ScrubDatasets::minimal_snaps_configuration, + /*num_osds */ 3}; + } +}; + +// some basic sanity checks +// (mainly testing the constructor) + +TEST_F(TestTScrubberBe_data_1, creation_1) +{ + /// \todo copy some osdmap tests from TestOSDMap.cc + ASSERT_TRUE(sbe); + ASSERT_TRUE(sbe->get_is_replicated()); + ASSERT_FALSE(sbe->get_m_repair()); + sbe->update_repair_status(true); + ASSERT_TRUE(sbe->get_m_repair()); + + // make sure *I* do not appear in 'all_but_me' set of OSDs + auto others = sbe->all_but_me(); + auto in_others = std::find(others.begin(), others.end(), i_am); + EXPECT_EQ(others.end(), in_others); +} + + +TEST_F(TestTScrubberBe_data_1, smaps_creation_1) +{ + ASSERT_TRUE(sbe); + ASSERT_EQ(sbe->get_omap_stats().omap_bytes, 0); + + // for test data 'minimal_snaps_configuration': + // scrub_compare_maps() should not emmit any error, nor + // return any snap-mapper fix + auto [incons, fix_list] = sbe->scrub_compare_maps(true, *test_scrubber); + + EXPECT_EQ(fix_list.size(), 0); // snap-mapper fix should be empty + + EXPECT_EQ(incons.size(), 0); // no inconsistency + + // make sure the test did execute *something* + EXPECT_TRUE(sbe->get_omap_stats().omap_bytes != 0); +} + + +// whitebox testing (OK if failing after a change to the backend internals) + + +// blackbox testing - testing the published functionality +// (should not depend on internals of the backend) + + +/// corrupt the snap_mapper data +TEST_F(TestTScrubberBe_data_1, snapmapper_1) +{ + ASSERT_TRUE(sbe); + + // a bogus version of hobj_ms1_snp30 (a clone) snap_ids + hobject_t hobj_ms1_snp30_inpool = hobject_t{ScrubDatasets::hobj_ms1_snp30}; + hobj_ms1_snp30_inpool.pool = pool_id; + all_clones_snaps_t bogus_30; + bogus_30[hobj_ms1_snp30_inpool] = {0x333, 0x666}; + + test_scrubber->set_snaps(bogus_30); + auto [incons, fix_list] = sbe->scrub_compare_maps(true, *test_scrubber); + + EXPECT_EQ(fix_list.size(), 1); + + // debug - print the fix-list: + for (const auto& fix : fix_list) { + std::cout << fmt::format("snapmapper_1: fix {}: {} {}->{}", + fix.hoid, + (fix.op == snap_mapper_op_t::add ? "add" : "upd"), + fix.wrong_snaps, + fix.snaps) + << std::endl; + } + EXPECT_EQ(fix_list[0].hoid, hobj_ms1_snp30_inpool); + EXPECT_EQ(fix_list[0].snaps, std::set{0x30}); + + EXPECT_EQ(incons.size(), 0); // no inconsistency +} + +// a dataset similar to 'minimal_snaps_configuration', +// but with the hobj_ms1_snp30 clone being modified by a corruption +// function +class TestTScrubberBe_data_2 : public TestTScrubberBe { + public: + TestTScrubberBe_data_2() : TestTScrubberBe() {} + + // basic test configuration - 3 OSDs, all involved in the pool + pool_conf_t pl{3, 3, 3, 3, "rep_pool"}; + + TestTScrubberBeParams inject_params() override + { + std::cout << fmt::format( + "{}: injecting params (minimal-snaps + size change)", + __func__) + << std::endl; + TestTScrubberBeParams params{ + /* pool_conf */ pl, + /* real_objs_conf */ ScrubDatasets::minimal_snaps_configuration, + /*num_osds */ 3}; + + // inject a corruption function that will modify osd.0's version of + // the object + params.objs_conf.objs[0].corrupt_funcs = &ScrubDatasets::crpt_funcs_set1; + return params; + } +}; + +TEST_F(TestTScrubberBe_data_2, smaps_clone_size) +{ + ASSERT_TRUE(sbe); + EXPECT_EQ(sbe->get_omap_stats().omap_bytes, 0); + logger.set_expected_err_count(1); + auto [incons, fix_list] = sbe->scrub_compare_maps(true, *test_scrubber); + + EXPECT_EQ(fix_list.size(), 0); // snap-mapper fix should be empty + + EXPECT_EQ(incons.size(), 1); // one inconsistency +} + +// Local Variables: +// compile-command: "cd ../.. ; make unittest_osdscrub ; ./unittest_osdscrub +// --log-to-stderr=true --debug-osd=20 # --gtest_filter=*.* " End: