From 2b6e1dd90b73bf1db16f9ee1b7ad519be3ca97ba Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Sun, 7 Jul 2024 12:46:25 -0500 Subject: [PATCH] osd/scrub: introducing the concept of a SchedEntry SchedEntry holds the scheduling details for scrubbing a specific PG at a specific scrub level. Namely - it identifies the [pg,level] combination, the 'urgency' attribute of the scheduled scrub (which determines most of its behavior and scheduling decisions) and the actual time attributes for scheduling (target, deadline, not_before). Added a table detailing, for each type of scrub, what limitations apply to it, and what restrictions are waived. The following commits will reshape the ScrubJob objects to hold two instances of SchedTarget-s - two wrappers around SchedEntry-s, one for the next shallow scrub and one for the next deep scrub. Sched-entries (wrapped in sched-targets) have a defined order: For ready-to-scrub entries (those that have an n.b. in the past), the order is first by urgency, then by target time (and then by level - deep before shallow - and then by the n.b. itself). 'Future' entries are ordered by n.b., then urgency, target time, and level. Signed-off-by: Ronen Friedman --- src/osd/scrubber/scrub_job.h | 30 +--- src/osd/scrubber/scrub_queue_entry.h | 231 +++++++++++++++++++++++++++ src/osd/scrubber_common.h | 59 +++++++ 3 files changed, 291 insertions(+), 29 deletions(-) create mode 100644 src/osd/scrubber/scrub_queue_entry.h diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h index ef30bcb4fe572..3b0fb8cb8b251 100644 --- a/src/osd/scrubber/scrub_job.h +++ b/src/osd/scrubber/scrub_job.h @@ -14,6 +14,7 @@ #include "osd/osd_types.h" #include "osd/osd_types_fmt.h" #include "osd/scrubber_common.h" +#include "scrub_queue_entry.h" /** * The ID used to name a candidate to scrub: @@ -27,23 +28,6 @@ namespace Scrub { enum class must_scrub_t { not_mandatory, mandatory }; -struct scrub_schedule_t { - utime_t scheduled_at{}; - utime_t deadline{0, 0}; - utime_t not_before{utime_t::max()}; - // when compared - the 'not_before' is ignored, assuming - // we never compare jobs with different eligibility status. - std::partial_ordering operator<=>(const scrub_schedule_t& rhs) const - { - auto cmp1 = scheduled_at <=> rhs.scheduled_at; - if (cmp1 != 0) { - return cmp1; - } - return deadline <=> rhs.deadline; - }; - bool operator==(const scrub_schedule_t& rhs) const = default; -}; - struct sched_params_t { utime_t proposed_time{}; must_scrub_t is_must{must_scrub_t::not_mandatory}; @@ -290,16 +274,4 @@ struct formatter { cf.mandatory_on_invalid); } }; - -template <> -struct formatter { - constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } - template - auto format(const Scrub::scrub_schedule_t& sc, FormatContext& ctx) const - { - return fmt::format_to( - ctx.out(), "nb:{:s}(at:{:s},dl:{:s})", sc.not_before, - sc.scheduled_at, sc.deadline); - } -}; } // namespace fmt diff --git a/src/osd/scrubber/scrub_queue_entry.h b/src/osd/scrubber/scrub_queue_entry.h new file mode 100644 index 0000000000000..9ab8affc601af --- /dev/null +++ b/src/osd/scrubber/scrub_queue_entry.h @@ -0,0 +1,231 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#pragma once + +#include +#include + +#include "include/utime.h" +#include "osd/osd_types.h" +#include "osd/scrubber_common.h" + +namespace Scrub { + +/** + * Possible urgency levels for a specific scheduling target (shallow or deep): + * + * (note: the 'urgency' attribute conveys both the relative priority for + * scheduling and the behavior of the scrub). The urgency levels are: + * ^^^^^^^^^^^^^^^^^^^^^ + * + * periodic scrubs: + * --------------- + * + * 'periodic_regular' - the "standard" shallow/deep scrub performed + * periodically on each PG. + * + * + * priority scrubs (termed 'required' or 'must' in the legacy code): + * --------------------------------------------------------------- + * In order of ascending priority: + * + * 'must_scrub' - the PG info is not valid (i.e. we do not have a valid + * 'last-scrub' stamp). A high-priority shallow scrub is required. + * + * 'after_repair' - triggered immediately after a recovery process + * ('m_after_repair_scrub_required' was set). + * This type of scrub is always deep. + * (note: this urgency level is not implemented in this commit) + * + * 'operator_requested' - the target was manually requested for scrubbing by + * an administrator. + * + * 'must_repair' - the target is required to be deep-scrubbed with the + * repair flag set, as: + * - the scrub was initiated by a message specifying 'do_repair'; or + * or - a deep scrub is required after the previous scrub ended with errors. + */ +enum class urgency_t { + periodic_regular, + must_scrub, + after_repair, + operator_requested, + must_repair, +}; + +/** + * SchedEntry holds the scheduling details for scrubbing a specific PG at + * a specific scrub level. Namely - it identifies the [pg,level] combination, + * the 'urgency' attribute of the scheduled scrub (which determines most of + * its behavior and scheduling decisions) and the actual time attributes + * for scheduling (target, deadline, not_before). + * + * In this commit - the 'urgency' attribute is not fully used yet, and some + * of the scrub behavior is still controlled by the 'planned scrub' flags. + */ +struct SchedEntry { + constexpr SchedEntry(spg_t pgid, scrub_level_t level) + : pgid{pgid} + , level{level} + {} + + SchedEntry(const SchedEntry&) = default; + SchedEntry(SchedEntry&&) = default; + SchedEntry& operator=(const SchedEntry&) = default; + SchedEntry& operator=(SchedEntry&&) = default; + + spg_t pgid; + scrub_level_t level; + + urgency_t urgency{urgency_t::periodic_regular}; + + /// scheduled_at, not-before & the deadline times + Scrub::scrub_schedule_t schedule; + + /// either 'none', or the reason for the latest failure/delay (for + /// logging/reporting purposes) + delay_cause_t last_issue{delay_cause_t::none}; + + // note: is_high_priority() is temporary. Will be removed + // in a followup commit. + bool is_high_priority() const + { + return urgency != urgency_t::periodic_regular; + } +}; + + +static inline std::weak_ordering cmp_ripe_entries( + const Scrub::SchedEntry& l, + const Scrub::SchedEntry& r) noexcept +{ + // for 'higher is better' sub elements - the 'r.' is on the left + if (auto cmp = r.urgency <=> l.urgency; cmp != 0) { + return cmp; + } + // if we are comparing the two targets of the same PG, once both are + // ripe - the 'deep' scrub is considered 'higher' than the 'shallow' one. + if (l.pgid == r.pgid && r.level < l.level) { + return std::weak_ordering::less; + } + // the 'utime_t' operator<=> is 'partial_ordering', it seems. + if (auto cmp = std::weak_order( + double(l.schedule.scheduled_at), double(r.schedule.scheduled_at)); + cmp != 0) { + return cmp; + } + if (r.level < l.level) { + return std::weak_ordering::less; + } + if (auto cmp = std::weak_order( + double(l.schedule.not_before), double(r.schedule.not_before)); + cmp != 0) { + return cmp; + } + return std::weak_ordering::greater; +} + +static inline std::weak_ordering cmp_future_entries( + const Scrub::SchedEntry& l, + const Scrub::SchedEntry& r) noexcept +{ + if (auto cmp = std::weak_order( + double(l.schedule.not_before), double(r.schedule.not_before)); + cmp != 0) { + return cmp; + } + // for 'higher is better' sub elements - the 'r.' is on the left + if (auto cmp = r.urgency <=> l.urgency; cmp != 0) { + return cmp; + } + if (auto cmp = std::weak_order( + double(l.schedule.scheduled_at), double(r.schedule.scheduled_at)); + cmp != 0) { + return cmp; + } + if (r.level < l.level) { + return std::weak_ordering::less; + } + return std::weak_ordering::greater; +} + +static inline std::weak_ordering cmp_entries( + utime_t t, + const Scrub::SchedEntry& l, + const Scrub::SchedEntry& r) noexcept +{ + bool l_ripe = l.schedule.not_before <= t; + bool r_ripe = r.schedule.not_before <= t; + if (l_ripe) { + if (r_ripe) { + return cmp_ripe_entries(l, r); + } + return std::weak_ordering::less; + } + if (r_ripe) { + return std::weak_ordering::greater; + } + return cmp_future_entries(l, r); +} + +// --- the interface required by 'not_before_queue_t': + +static inline const utime_t& project_not_before(const Scrub::SchedEntry& e) +{ + return e.schedule.not_before; +} + +static inline const spg_t& project_removal_class(const Scrub::SchedEntry& e) +{ + return e.pgid; +} + + +/// 'not_before_queue_t' requires a '<' operator, to be used for +/// eligible entries: +static inline bool operator<( + const Scrub::SchedEntry& lhs, + const Scrub::SchedEntry& rhs) +{ + return cmp_ripe_entries(lhs, rhs) == std::weak_ordering::less; +} + +} // namespace Scrub + + +namespace fmt { + +// clang-format off +template <> +struct formatter : formatter { + template + auto format(Scrub::urgency_t urg, FormatContext& ctx) const + { + using enum Scrub::urgency_t; + std::string_view desc; + switch (urg) { + case periodic_regular: desc = "periodic-regular"; break; + case must_scrub: desc = "must-scrub"; break; + case after_repair: desc = "after-repair"; break; + case operator_requested: desc = "operator-requested"; break; + case must_repair: desc = "must-repair"; break; + // better to not have a default case, so that the compiler will warn + } + return formatter::format(desc, ctx); + } +}; +// clang-format on + +template <> +struct formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + template + auto format(const Scrub::SchedEntry& st, FormatContext& ctx) const + { + return fmt::format_to( + ctx.out(), "{}/{},nb:{:s},({},tr:{:s},dl:{:s})", st.pgid, + (st.level == scrub_level_t::deep ? "dp" : "sh"), st.schedule.not_before, + st.urgency, st.schedule.scheduled_at, st.schedule.deadline); + } +}; +} // namespace fmt diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h index 832811fbad76f..d4648c68a037a 100644 --- a/src/osd/scrubber_common.h +++ b/src/osd/scrubber_common.h @@ -108,6 +108,52 @@ enum class schedule_result_t { osd_wide_failure // failed to scrub any target }; +/// a collection of the basic scheduling information of a scrub target: +/// target time to scrub, the 'not before' time, and a deadline. +struct scrub_schedule_t { + /** + * the time at which we are allowed to start the scrub. Never + * decreasing after 'scheduled_at' is set. + */ + utime_t not_before{utime_t::max()}; + + /** + * the 'deadline' is the time by which we expect the periodic scrub to + * complete. It is determined by the SCRUB_MAX_INTERVAL pool configuration + * and by osd_scrub_max_interval; + * Once passed, the scrub will be allowed to run even if the OSD is + * overloaded.It would also have higher priority than other + * auto-scheduled scrubs. + */ + utime_t deadline{utime_t::max()}; + + /** + * the 'scheduled_at' is the time at which we intended the scrub to be scheduled. + * For periodic (regular) scrubs, it is set to the time of the last scrub + * plus the scrub interval (plus some randomization). Priority scrubs + * have their own specific rules for the target time. E.g.: + * - for operator-initiated scrubs: 'target' is set to 'scrub_must_stamp'; + * - same for re-scrubbing (deep scrub after a shallow scrub that ended with + * errors; + * - when requesting a scrub after a repair (the highest priority scrub): + * the target is set to '0' (beginning of time); + */ + utime_t scheduled_at{utime_t::max()}; + + std::partial_ordering operator<=>(const scrub_schedule_t& rhs) const + { + // when compared - the 'not_before' is ignored, assuming + // we never compare jobs with different eligibility status. + auto cmp1 = scheduled_at <=> rhs.scheduled_at; + if (cmp1 != 0) { + return cmp1; + } + return deadline <=> rhs.deadline; + }; + bool operator==(const scrub_schedule_t& rhs) const = default; +}; + + /// rescheduling param: should we delay jobs already ready to execute? enum class delay_ready_t : bool { delay_ready = true, no_delay = false }; @@ -145,6 +191,19 @@ struct formatter { conds.allow_requested_repair_only); } }; + +template <> +struct formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + template + auto format(const Scrub::scrub_schedule_t& sc, FormatContext& ctx) const + { + return fmt::format_to( + ctx.out(), "nb:{:s}(at:{:s},dl:{:s})", sc.not_before, + sc.scheduled_at, sc.deadline); + } +}; + } // namespace fmt namespace Scrub { -- 2.39.5