From: Ronen Friedman Date: Fri, 9 May 2025 12:46:26 +0000 (-0500) Subject: osd/scrub: remove the deep-scrubs deadline attribute X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4e6323f894fcd96a3d6235451a7d54913e6a1d40;p=ceph.git osd/scrub: remove the deep-scrubs deadline attribute As it is no longer meaningful in the context of the new scrub scheduling design. The change mandates fixes to the way 'schedule-[deeps]crub' commands are implemented. The offset to use when forcing the last-scrub timestamp to a new value in now calculated in ScrubJob::guaranteed_offset(), as ScrubJob is where all schedule adjustments (which employ the same logic) are implemented. Signed-off-by: Ronen Friedman --- diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index 5f7e2733263b2..28e94e54631c3 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -687,14 +687,15 @@ Scrub::sched_conf_t PgScrubber::populate_config_params() const deep_pool > 0.0 ? deep_pool : conf->osd_deep_scrub_interval; /** - * 'max_deep' and 'max_shallow' are set to the maximum allowed delay between - * scrubs. These deadlines have almost no effect on scrub scheduling + * 'max_shallow' is set to the maximum allowed delay between + * scrubs. This deadline has almost no effect on scrub scheduling * (the only minor exception: when sorting two scrub jobs that are - * equivalent in all but the deadline). + * equivalent in all but the deadline). It will be removed in + * the next version. * * 'max_shallow' is controlled by a pool option and a configuration * parameter. Note that if the value configured is less than the - * shallow interval, the max_shallow is disabled. + * shallow interval (plus expenses), the max_shallow is disabled. */ auto max_shallow = pool_conf.value_or(pool_opts_t::SCRUB_MAX_INTERVAL, 0.0); if (max_shallow <= 0.0) { @@ -721,10 +722,6 @@ Scrub::sched_conf_t PgScrubber::populate_config_params() const } } - // There are no comparable options for max_deep. We set it here to - // 4X the deep interval, as a reasonable default. - configs.max_deep = 4 * configs.deep_interval; - configs.interval_randomize_ratio = conf->osd_scrub_interval_randomize_ratio; configs.deep_randomize_ratio = conf.get_val("osd_deep_scrub_interval_cv"); @@ -762,33 +759,29 @@ void PgScrubber::on_operator_periodic_cmd( scrub_level_t scrub_level, int64_t offset) { - const auto cnf = populate_config_params(); - dout(10) << fmt::format( - "{}: {} (cmd offset:{}) conf:{}", __func__, - (scrub_level == scrub_level_t::deep ? "deep" : "shallow"), offset, - cnf) - << dendl; - - // move the relevant time-stamp backwards - enough to trigger a scrub - utime_t stamp = ceph_clock_now(); - if (offset > 0) { - stamp -= offset; + // if 'offset' wasn't specified - find a value that guarantees the scrub + // target will appear ready for scrubbing (even after random adjustments) + if (offset == 0) { + const auto cnf = populate_config_params(); + offset = m_scrub_job->guaranteed_offset(scrub_level, cnf); + dout(15) << fmt::format( + "{}: {} (calculated offset:{}) conf:{}", __func__, + (scrub_level == scrub_level_t::deep ? "deep" : "shallow"), + offset, cnf) + << dendl; } else { - double max_iv = - (scrub_level == scrub_level_t::deep) - ? 2 * cnf.max_deep - : (cnf.max_shallow ? *cnf.max_shallow : cnf.shallow_interval); - dout(20) << fmt::format( - "{}: stamp:{:s} ms:{}/{}/{}", __func__, stamp, - (cnf.max_shallow ? "ms+" : "ms-"), - (cnf.max_shallow ? *cnf.max_shallow : -999.99), - cnf.shallow_interval) + dout(15) << fmt::format( + "{}: {} command offset:{}", __func__, + (scrub_level == scrub_level_t::deep ? "deep" : "shallow"), + offset) << dendl; - stamp -= max_iv; } - stamp -= 100.0; // for good measure + // move the relevant time-stamp backwards - enough to trigger a scrub + utime_t stamp = ceph_clock_now(); + stamp -= offset; // can't combine with prev line - dout(10) << fmt::format("{}: stamp:{:s} ", __func__, stamp) << dendl; + dout(10) << fmt::format("{}: calculated stamp:{:s}", __func__, stamp) + << dendl; asok_response_section(f, true, scrub_level, stamp); if (scrub_level == scrub_level_t::deep) { diff --git a/src/osd/scrubber/scrub_job.cc b/src/osd/scrubber/scrub_job.cc index ffd0381d96cc5..45b009c65e230 100644 --- a/src/osd/scrubber/scrub_job.cc +++ b/src/osd/scrubber/scrub_job.cc @@ -146,6 +146,22 @@ void ScrubJob::adjust_shallow_schedule( } +double ScrubJob::guaranteed_offset( + scrub_level_t s_or_d, + const Scrub::sched_conf_t& app_conf) +{ + if (s_or_d == scrub_level_t::deep) { + // use the sdv of the deep scrub distribution, times 3 (3-sigma...) + const double sdv = app_conf.deep_interval * app_conf.deep_randomize_ratio; + // note: the '+10.0' is there just to guarantee inequality if '._ratio' is 0 + return app_conf.deep_interval + abs(3 * sdv) + 10.0; + } + + // shallow scrub + return app_conf.shallow_interval * (2.0 + app_conf.interval_randomize_ratio); +} + + void ScrubJob::operator_forced(scrub_level_t s_or_d, scrub_type_t scrub_type) { auto& trgt = get_target(s_or_d); @@ -237,44 +253,35 @@ void ScrubJob::adjust_deep_schedule( << dendl; auto& dp_times = deep_target.sched_info.schedule; // shorthand + dp_times.deadline = utime_t::max(); // no 'max' for deep scrubs if (ScrubJob::requires_randomization(deep_target.urgency())) { - utime_t adj_not_before = last_deep; utime_t adj_target = last_deep; - dp_times.deadline = adj_target; // add a random delay to the proposed scheduled time const double sdv = app_conf.deep_interval * app_conf.deep_randomize_ratio; std::normal_distribution normal_dist{app_conf.deep_interval, sdv}; - auto next_delay = - std::clamp(normal_dist(random_gen), app_conf.deep_interval - 2 * sdv, - app_conf.deep_interval + 2 * sdv); + auto next_delay = std::clamp( + normal_dist(random_gen), app_conf.deep_interval - 2 * sdv, + app_conf.deep_interval + 2 * sdv); adj_target += next_delay; dout(20) << fmt::format( - "deep scrubbing: next_delay={:.0f} (interval={:.0f}, " - "ratio={:.3f}), adjusted:{:s}", - next_delay, app_conf.deep_interval, - app_conf.deep_randomize_ratio, adj_target) - << dendl; - - dp_times.deadline += app_conf.max_deep; + "deep scrubbing: next_delay={:.0f} (interval={:.0f}, " + "ratio={:.3f}), adjusted:{:s}", + next_delay, app_conf.deep_interval, + app_conf.deep_randomize_ratio, adj_target) + << dendl; - if (adj_not_before < adj_target) { - adj_not_before = adj_target; - } dp_times.scheduled_at = adj_target; - dp_times.not_before = adj_not_before; + dp_times.not_before = adj_target; } else { - // the target time is already set. Make sure to reset the n.b. and - // the (irrelevant) deadline + // the target time is already set. The n.b. is set to same dp_times.not_before = dp_times.scheduled_at; - dp_times.deadline = utime_t::max(); } dout(10) << fmt::format( - "adjusted: nb:{:s} target:{:s} deadline:{:s} ({})", - dp_times.not_before, dp_times.scheduled_at, dp_times.deadline, - state_desc()) + "adjusted: nb:{:s} target:{:s} ({})", dp_times.not_before, + dp_times.scheduled_at, state_desc()) << dendl; } diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h index 2c2d38ec5380a..935aa7135a857 100644 --- a/src/osd/scrubber/scrub_job.h +++ b/src/osd/scrubber/scrub_job.h @@ -38,19 +38,12 @@ struct sched_conf_t { double deep_interval{0.0}; /** - * the maximum interval between shallow scrubs, as determined by either the - * OSD or the pool configuration. Empty if no limit is configured. - */ - std::optional max_shallow; - - /** - * the maximum interval between deep scrubs, after which the + * the maximum interval between shallow scrubs, after which the * (info-only) "overdue" field in the scheduler dump is set. - * There is no specific configuration parameter to control the - * deep scrubs max. Instead - we set it to 4 times the average - * interval. + * Determined by either the pool or the cluster configuration. + * Empty if no limit is configured. */ - double max_deep{std::numeric_limits::max()}; + std::optional max_shallow; /** * interval_randomize_ratio @@ -274,6 +267,17 @@ class ScrubJob { */ void operator_forced(scrub_level_t s_or_d, scrub_type_t scrub_type); + /** + * calculate a time offset large enough, so that once the relevant + * last-scrub timestamp is forced back by this amount, the PG is + * eligible for a periodic scrub of the specified level. + * Used by the scrubber upon receiving a 'fake a scheduled scrub' request + * from the operator. + */ + double guaranteed_offset( + scrub_level_t s_or_d, + const Scrub::sched_conf_t& app_conf); + void dump(ceph::Formatter* f) const; bool is_registered() const { return registered; } @@ -430,9 +434,9 @@ struct formatter { { return fmt::format_to( ctx.out(), - "periods:s:{}/{},d:{}/{},iv-ratio:{},deep-rand:{},on-inv:{}", + "periods:s:{}/{},d:{},iv-ratio:{},deep-rand:{},on-inv:{}", cf.shallow_interval, cf.max_shallow.value_or(-1.0), cf.deep_interval, - cf.max_deep, cf.interval_randomize_ratio, cf.deep_randomize_ratio, + cf.interval_randomize_ratio, cf.deep_randomize_ratio, cf.mandatory_on_invalid); } };