From 38d9cf4ca760c667d105435a714f76dbff926960 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Mon, 23 Sep 2024 01:59:30 -0500 Subject: [PATCH] osd/scrub: introduce ScrubStore::at_level_t to hold the caching and backend details related to the representation of scrub-detected errors as OMap entries of a uniquely-named object. In a followup commit - the ScrubStore is modified to hold two of these objects, one for the shallow errors and one for the deep errors. Signed-off-by: Ronen Friedman --- src/osd/scrubber/ScrubStore.cc | 49 +++++++++++++++++++++---------- src/osd/scrubber/ScrubStore.h | 53 +++++++++++++++++++++++++++++----- 2 files changed, 79 insertions(+), 23 deletions(-) diff --git a/src/osd/scrubber/ScrubStore.cc b/src/osd/scrubber/ScrubStore.cc index a00ab2caece..af223cb5cdc 100644 --- a/src/osd/scrubber/ScrubStore.cc +++ b/src/osd/scrubber/ScrubStore.cc @@ -109,19 +109,29 @@ Store::create(ObjectStore* store, ceph_assert(t); ghobject_t oid = make_scrub_object(pgid); t->touch(coll, oid); - return new Store{coll, oid, store}; + return new Store{*store, t, pgid, coll}; +} + + +Store::Store( + ObjectStore& osd_store, + ObjectStore::Transaction* t, + const spg_t& pgid, + const coll_t& coll) + : object_store{osd_store} + , coll{coll} +{ + ceph_assert(t); + + const auto err_obj = pgid.make_temp_ghobject(fmt::format("scrub_{}", pgid)); + t->touch(coll, err_obj); + errors_db.emplace(pgid, err_obj, OSDriver{&object_store, coll, err_obj}); } -Store::Store(const coll_t& coll, const ghobject_t& oid, ObjectStore* store) - : coll(coll), - hoid(oid), - driver(store, coll, hoid), - backend(&driver) -{} Store::~Store() { - ceph_assert(results.empty()); + ceph_assert(!errors_db || errors_db->results.empty()); } void Store::add_error(int64_t pool, const inconsistent_obj_wrapper& e) @@ -131,11 +141,13 @@ void Store::add_error(int64_t pool, const inconsistent_obj_wrapper& e) void Store::add_object_error(int64_t pool, const inconsistent_obj_wrapper& e) { + const auto key = to_object_key(pool, e.object); bufferlist bl; e.encode(bl); - results[to_object_key(pool, e.object)] = bl; + errors_db->results[key] = bl; } + void Store::add_error(int64_t pool, const inconsistent_snapset_wrapper& e) { add_snap_error(pool, e); @@ -145,26 +157,28 @@ void Store::add_snap_error(int64_t pool, const inconsistent_snapset_wrapper& e) { bufferlist bl; e.encode(bl); - results[to_snap_key(pool, e.object)] = bl; + errors_db->results[to_snap_key(pool, e.object)] = bl; } bool Store::empty() const { - return results.empty(); + return errors_db->results.empty(); } void Store::flush(ObjectStore::Transaction* t) { if (t) { - OSDriver::OSTransaction txn = driver.get_transaction(t); - backend.set_keys(results, &txn); + OSDriver::OSTransaction txn = errors_db->driver.get_transaction(t); + errors_db->backend.set_keys(errors_db->results, &txn); } - results.clear(); + errors_db->results.clear(); } void Store::cleanup(ObjectStore::Transaction* t) { - t->remove(coll, hoid); + ceph_assert(t); + if (errors_db) + t->remove(coll, errors_db->errors_hoid); } std::vector @@ -195,8 +209,11 @@ Store::get_errors(const string& begin, uint64_t max_return) const { vector errors; + if (!errors_db) + return errors; + auto next = std::make_pair(begin, bufferlist{}); - while (max_return && !backend.get_next(next.first, &next)) { + while (max_return && !errors_db->backend.get_next(next.first, &next)) { if (next.first >= end) break; errors.push_back(next.second); diff --git a/src/osd/scrubber/ScrubStore.h b/src/osd/scrubber/ScrubStore.h index 567badf608b..949a976051e 100644 --- a/src/osd/scrubber/ScrubStore.h +++ b/src/osd/scrubber/ScrubStore.h @@ -5,6 +5,7 @@ #define CEPH_SCRUB_RESULT_H #include "common/map_cacher.hpp" +#include "osd/osd_types_fmt.h" #include "osd/SnapMapper.h" // for OSDriver namespace librados { @@ -45,18 +46,56 @@ class Store { uint64_t max_return) const; private: - Store(const coll_t& coll, const ghobject_t& oid, ObjectStore* store); + /** + * at_level_t + * + * The machinery for caching and storing errors at a specific scrub level. + */ + struct at_level_t { + at_level_t(const spg_t& pgid, const ghobject_t& err_obj, OSDriver&& drvr) + : errors_hoid{err_obj} + , driver{std::move(drvr)} + , backend{&driver} + {} + + /// the object in the PG store, where the errors are stored + ghobject_t errors_hoid; + + /// abstracted key fetching + OSDriver driver; + + /// a K,V cache for the errors that are detected during the scrub + /// session. The errors marked for a specific object are stored as + /// an OMap entry with the object's name as the key. + MapCacher::MapCacher backend; + + /// a temp object mapping seq-id to inconsistencies + std::map results; + }; + + Store(ObjectStore& osd_store, + ObjectStore::Transaction* t, + const spg_t& pgid, + const coll_t& coll); + std::vector get_errors(const std::string& start, const std::string& end, uint64_t max_return) const; private: + /// the OSD's storage backend + ObjectStore& object_store; + + /// the collection (i.e. - the PG store) in which the errors are stored const coll_t coll; - const ghobject_t hoid; - // a temp object holding mappings from seq-id to inconsistencies found in - // scrubbing - OSDriver driver; - mutable MapCacher::MapCacher backend; - std::map results; + + /** + * the machinery (backend details, cache, etc.) for storing both levels + * of errors (note: 'optional' to allow delayed creation w/o dynamic + * allocations; and 'mutable', as the caching mechanism is used in const + * methods) + */ + mutable std::optional errors_db; + // not yet: mutable std::optional deep_db; }; } // namespace Scrub -- 2.39.5