From: Ronen Friedman Date: Mon, 10 Jun 2024 12:25:11 +0000 (-0500) Subject: osd/scrub: relocating scrub-related PG methods to PgScrubber.cc X-Git-Tag: v20.0.0~1720^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d89f9ccabf2d208c4de9764ec0f904626f9f6a1d;p=ceph.git osd/scrub: relocating scrub-related PG methods to PgScrubber.cc No code changes in this commit. The methods moved are still 'PG::'. Signed-off-by: Ronen Friedman --- diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 30aec9045381..24de1b3f6488 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -1364,311 +1364,6 @@ double PG::next_deepscrub_interval() const deep_scrub_interval = cct->_conf->osd_deep_scrub_interval; return info.history.last_deep_scrub_stamp + deep_scrub_interval; } - -bool PG::is_time_for_deep(bool allow_deep_scrub, - bool allow_shallow_scrub, - bool has_deep_errors, - const requested_scrub_t& planned) const -{ - dout(10) << fmt::format( - "{}: need-auto? {} allowed? {}/{} deep-errors? {} " - "last_deep_scrub_stamp {}", - __func__, - planned.need_auto, - allow_shallow_scrub, - allow_deep_scrub, - has_deep_errors, - info.history.last_deep_scrub_stamp) - << dendl; - - if (!allow_deep_scrub) - return false; - - if (planned.need_auto) { - dout(10) << __func__ << ": need repair after scrub errors" << dendl; - return true; - } - - if (ceph_clock_now() >= next_deepscrub_interval()) { - dout(20) << __func__ << ": now (" << ceph_clock_now() - << ") >= time for deep (" << next_deepscrub_interval() << ")" - << dendl; - return true; - } - - if (has_deep_errors) { - // note: the text below is matched by 'standalone' tests - osd->clog->info() << "osd." << osd->whoami << " pg " << info.pgid - << " Deep scrub errors, upgrading scrub to deep-scrub"; - return true; - } - - // we only flip coins if 'allow_shallow_scrub' is asserted. Otherwise - as - // this function is called often, we will probably be deep-scrubbing most of - // the time. - if (allow_shallow_scrub) { - const bool deep_coin_flip = - (rand() % 100) < cct->_conf->osd_deep_scrub_randomize_ratio * 100; - - dout(15) << __func__ << ": time_for_deep=" << planned.time_for_deep - << " deep_coin_flip=" << deep_coin_flip << dendl; - - if (deep_coin_flip) - return true; - } - - return false; -} - -/* - clang-format off - - Request details | none | no-scrub | no-scrub+no-deep | no-deep - ------------------------------------------------------------------------ - ------------------------------------------------------------------------ - initiated | shallow | shallow | shallow | shallow - ------------------------------------------------------------------------ - init. + t.f.deep | deep | deep | shallow | shallow - ------------------------------------------------------------------------ - initiated deep | deep | deep | deep | deep - ------------------------------------------------------------------------ - - clang-format on -*/ -std::optional PG::validate_initiated_scrub( - bool allow_deep_scrub, - bool try_to_auto_repair, - bool time_for_deep, - bool has_deep_errors, - const requested_scrub_t& planned) const -{ - requested_scrub_t upd_flags{planned}; - - upd_flags.time_for_deep = time_for_deep; - upd_flags.deep_scrub_on_error = false; - upd_flags.auto_repair = false; - - if (upd_flags.must_deep_scrub) { - upd_flags.calculated_to_deep = true; - } else if (upd_flags.time_for_deep && allow_deep_scrub) { - upd_flags.calculated_to_deep = true; - } else { - upd_flags.calculated_to_deep = false; - if (has_deep_errors) { - osd->clog->error() << fmt::format( - "osd.{} pg {} Regular scrub request, deep-scrub details will be lost", - osd->whoami, - info.pgid); - } - } - - if (try_to_auto_repair) { - // for shallow scrubs: rescrub if errors found - // for deep: turn 'auto-repair' on - if (upd_flags.calculated_to_deep) { - dout(10) << fmt::format( - "{}: performing an auto-repair deep scrub", - __func__) - << dendl; - upd_flags.auto_repair = true; - } else { - dout(10) << fmt::format( - "{}: will perform an auto-repair deep scrub if errors " - "are found", - __func__) - << dendl; - upd_flags.deep_scrub_on_error = true; - } - } - - return upd_flags; -} - -/* - clang-format off - - for periodic scrubs: - - Periodic type | none | no-scrub | no-scrub+no-deep | no-deep - ------------------------------------------------------------------------ - ------------------------------------------------------------------------ - periodic | shallow | x | x | shallow - ------------------------------------------------------------------------ - periodic + t.f.deep| deep | deep | x | shallow - ------------------------------------------------------------------------ - - clang-format on -*/ -std::optional PG::validate_periodic_mode( - bool allow_deep_scrub, - bool try_to_auto_repair, - bool allow_shallow_scrub, - bool time_for_deep, - bool has_deep_errors, - const requested_scrub_t& planned) const - -{ - ceph_assert(!planned.must_deep_scrub && !planned.must_repair); - - if (!allow_deep_scrub && has_deep_errors) { - osd->clog->error() - << "osd." << osd->whoami << " pg " << info.pgid - << " Regular scrub skipped due to deep-scrub errors and nodeep-scrub set"; - return std::nullopt; // no scrubbing - } - - requested_scrub_t upd_flags{planned}; - - upd_flags.time_for_deep = time_for_deep; - upd_flags.deep_scrub_on_error = false; - upd_flags.auto_repair = false; - upd_flags.calculated_to_deep = false; - - dout(20) << fmt::format("{}: allowed:{}/{} t.f.d:{} req:{}", - __func__, - allow_shallow_scrub, - allow_deep_scrub, - upd_flags.time_for_deep, - planned) - << dendl; - - // should we perform a shallow scrub? - if (allow_shallow_scrub) { - if (!upd_flags.time_for_deep || !allow_deep_scrub) { - if (try_to_auto_repair) { - dout(10) << __func__ - << ": auto repair with scrubbing, rescrub if errors found" - << dendl; - upd_flags.deep_scrub_on_error = true; - } - dout(20) << __func__ << " will do shallow scrub (time_for_deep = " - << upd_flags.time_for_deep << ")" << dendl; - return upd_flags; - } - // else - either deep-scrub or nothing - } - - if (upd_flags.time_for_deep) { - if (allow_deep_scrub) { - if (try_to_auto_repair) { - dout(20) << __func__ << ": auto repair with deep scrubbing" << dendl; - upd_flags.auto_repair = true; - } - upd_flags.calculated_to_deep = true; - dout(20) << fmt::format("{}: final: {}", __func__, upd_flags) << dendl; - return upd_flags; - } - if (allow_shallow_scrub) { - dout(20) << fmt::format("{}: final:{}", __func__, upd_flags) << dendl; - return upd_flags; - } - return std::nullopt; - } - - return std::nullopt; // no scrubbing -} - - -/* - From docs.ceph.com (osd-internals/scrub): - - clang-format off - - Desired no-scrub flags & scrub type interactions: - - Periodic type | none | no-scrub | no-scrub+no-deep | no-deep - ------------------------------------------------------------------------ - ------------------------------------------------------------------------ - periodic | shallow | x | x | shallow - ------------------------------------------------------------------------ - periodic + t.f.deep| deep | deep | x | shallow - ------------------------------------------------------------------------ - initiated | shallow | shallow | shallow | shallow - ------------------------------------------------------------------------ - init. + t.f.deep | deep | deep | shallow | shallow - ------------------------------------------------------------------------ - initiated deep | deep | deep | deep | deep - ------------------------------------------------------------------------ - - "periodic" - if !must_scrub && !must_deep_scrub; - "initiated deep" - if must_scrub && must_deep_scrub; - "initiated" - if must_scrub && !must_deep_scrub; - - clang-format on -*/ -/* - * The returned flags collection (requested_scrub_t) is based on - * m_planned_scrub with the following modifications: - * - * - calculated_to_deep will be set to shallow or deep, depending on the - * scrub type (according to the decision table above); - * - deep_scrub_on_error will be determined; - * - same for auto_repair; - * - time_for_deep will be set to true if the scrub is periodic and the - * time for a deep scrub has been reached (+ some other conditions); - * and - * - need_auto is cleared - */ -std::optional PG::validate_scrub_mode() const -{ - const bool allow_shallow_scrub = - !(get_osdmap()->test_flag(CEPH_OSDMAP_NOSCRUB) || - pool.info.has_flag(pg_pool_t::FLAG_NOSCRUB)); - const bool allow_deep_scrub = - !(get_osdmap()->test_flag(CEPH_OSDMAP_NODEEP_SCRUB) || - pool.info.has_flag(pg_pool_t::FLAG_NODEEP_SCRUB)); - const bool has_deep_errors = (info.stats.stats.sum.num_deep_scrub_errors > 0); - const bool try_to_auto_repair = (cct->_conf->osd_scrub_auto_repair && - get_pgbackend()->auto_repair_supported()); - - dout(10) << __func__ << " pg: " << info.pgid - << " allow: " << allow_shallow_scrub << "/" << allow_deep_scrub - << " deep errs: " << has_deep_errors - << " auto-repair: " << try_to_auto_repair << " (" - << cct->_conf->osd_scrub_auto_repair << ")" << dendl; - - // scrubbing while recovering? - const bool prevented_by_recovery = - osd->is_recovery_active() && !cct->_conf->osd_scrub_during_recovery && - (!cct->_conf->osd_repair_during_recovery || !m_planned_scrub.must_repair); - - if (prevented_by_recovery) { - dout(20) << __func__ << ": scrubbing prevented during recovery" << dendl; - return std::nullopt; - } - - const bool time_for_deep = is_time_for_deep(allow_deep_scrub, - allow_shallow_scrub, - has_deep_errors, - m_planned_scrub); - std::optional upd_flags; - - if (m_planned_scrub.must_scrub) { - upd_flags = validate_initiated_scrub(allow_deep_scrub, - try_to_auto_repair, - time_for_deep, - has_deep_errors, - m_planned_scrub); - } else { - ceph_assert(!m_planned_scrub.must_deep_scrub); - upd_flags = validate_periodic_mode(allow_deep_scrub, - try_to_auto_repair, - allow_shallow_scrub, - time_for_deep, - has_deep_errors, - m_planned_scrub); - if (!upd_flags) { - dout(20) << __func__ << ": no periodic scrubs allowed" << dendl; - return std::nullopt; - } - } - - dout(10) << fmt::format("{}: next scrub flags: {}", __func__, *upd_flags) - << dendl; - upd_flags->need_auto = false; - return upd_flags; -} - void PG::on_scrub_schedule_input_change() { if (is_active() && is_primary()) { diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index 75b690b6a58f..9cc115ff637d 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -2557,6 +2557,326 @@ void PgScrubber::update_scrub_stats(ceph::coarse_real_clock::time_point now_is) } } +///////////////////////// moved as-is from PG.cc ///////////////////////////// +///////////////////////// to be refactored later ///////////////////////////// + +#undef dout_context +#define dout_context cct +#undef dout_prefix +#define dout_prefix _pgs_prefix(_dout, this) +template +static ostream& _pgs_prefix(std::ostream *_dout, T *t) +{ + return t->gen_prefix(*_dout); +} + + + +bool PG::is_time_for_deep(bool allow_deep_scrub, + bool allow_shallow_scrub, + bool has_deep_errors, + const requested_scrub_t& planned) const +{ + dout(10) << fmt::format( + "{}: need-auto? {} allowed? {}/{} deep-errors? {} " + "last_deep_scrub_stamp {}", + __func__, + planned.need_auto, + allow_shallow_scrub, + allow_deep_scrub, + has_deep_errors, + info.history.last_deep_scrub_stamp) + << dendl; + + if (!allow_deep_scrub) + return false; + + if (planned.need_auto) { + dout(10) << __func__ << ": need repair after scrub errors" << dendl; + return true; + } + + if (ceph_clock_now() >= next_deepscrub_interval()) { + dout(20) << __func__ << ": now (" << ceph_clock_now() + << ") >= time for deep (" << next_deepscrub_interval() << ")" + << dendl; + return true; + } + + if (has_deep_errors) { + // note: the text below is matched by 'standalone' tests + osd->clog->info() << "osd." << osd->whoami << " pg " << info.pgid + << " Deep scrub errors, upgrading scrub to deep-scrub"; + return true; + } + + // we only flip coins if 'allow_shallow_scrub' is asserted. Otherwise - as + // this function is called often, we will probably be deep-scrubbing most of + // the time. + if (allow_shallow_scrub) { + const bool deep_coin_flip = + (rand() % 100) < cct->_conf->osd_deep_scrub_randomize_ratio * 100; + + dout(15) << __func__ << ": time_for_deep=" << planned.time_for_deep + << " deep_coin_flip=" << deep_coin_flip << dendl; + + if (deep_coin_flip) + return true; + } + + return false; +} + +/* + clang-format off + + Request details | none | no-scrub | no-scrub+no-deep | no-deep + ------------------------------------------------------------------------ + ------------------------------------------------------------------------ + initiated | shallow | shallow | shallow | shallow + ------------------------------------------------------------------------ + init. + t.f.deep | deep | deep | shallow | shallow + ------------------------------------------------------------------------ + initiated deep | deep | deep | deep | deep + ------------------------------------------------------------------------ + + clang-format on +*/ +std::optional PG::validate_initiated_scrub( + bool allow_deep_scrub, + bool try_to_auto_repair, + bool time_for_deep, + bool has_deep_errors, + const requested_scrub_t& planned) const +{ + requested_scrub_t upd_flags{planned}; + + upd_flags.time_for_deep = time_for_deep; + upd_flags.deep_scrub_on_error = false; + upd_flags.auto_repair = false; + + if (upd_flags.must_deep_scrub) { + upd_flags.calculated_to_deep = true; + } else if (upd_flags.time_for_deep && allow_deep_scrub) { + upd_flags.calculated_to_deep = true; + } else { + upd_flags.calculated_to_deep = false; + if (has_deep_errors) { + osd->clog->error() << fmt::format( + "osd.{} pg {} Regular scrub request, deep-scrub details will be lost", + osd->whoami, + info.pgid); + } + } + + if (try_to_auto_repair) { + // for shallow scrubs: rescrub if errors found + // for deep: turn 'auto-repair' on + if (upd_flags.calculated_to_deep) { + dout(10) << fmt::format( + "{}: performing an auto-repair deep scrub", + __func__) + << dendl; + upd_flags.auto_repair = true; + } else { + dout(10) << fmt::format( + "{}: will perform an auto-repair deep scrub if errors " + "are found", + __func__) + << dendl; + upd_flags.deep_scrub_on_error = true; + } + } + + return upd_flags; +} + +/* + clang-format off + + for periodic scrubs: + + Periodic type | none | no-scrub | no-scrub+no-deep | no-deep + ------------------------------------------------------------------------ + ------------------------------------------------------------------------ + periodic | shallow | x | x | shallow + ------------------------------------------------------------------------ + periodic + t.f.deep| deep | deep | x | shallow + ------------------------------------------------------------------------ + + clang-format on +*/ +std::optional PG::validate_periodic_mode( + bool allow_deep_scrub, + bool try_to_auto_repair, + bool allow_shallow_scrub, + bool time_for_deep, + bool has_deep_errors, + const requested_scrub_t& planned) const + +{ + ceph_assert(!planned.must_deep_scrub && !planned.must_repair); + + if (!allow_deep_scrub && has_deep_errors) { + osd->clog->error() + << "osd." << osd->whoami << " pg " << info.pgid + << " Regular scrub skipped due to deep-scrub errors and nodeep-scrub set"; + return std::nullopt; // no scrubbing + } + + requested_scrub_t upd_flags{planned}; + + upd_flags.time_for_deep = time_for_deep; + upd_flags.deep_scrub_on_error = false; + upd_flags.auto_repair = false; + upd_flags.calculated_to_deep = false; + + dout(20) << fmt::format("{}: allowed:{}/{} t.f.d:{} req:{}", + __func__, + allow_shallow_scrub, + allow_deep_scrub, + upd_flags.time_for_deep, + planned) + << dendl; + + // should we perform a shallow scrub? + if (allow_shallow_scrub) { + if (!upd_flags.time_for_deep || !allow_deep_scrub) { + if (try_to_auto_repair) { + dout(10) << __func__ + << ": auto repair with scrubbing, rescrub if errors found" + << dendl; + upd_flags.deep_scrub_on_error = true; + } + dout(20) << __func__ << " will do shallow scrub (time_for_deep = " + << upd_flags.time_for_deep << ")" << dendl; + return upd_flags; + } + // else - either deep-scrub or nothing + } + + if (upd_flags.time_for_deep) { + if (allow_deep_scrub) { + if (try_to_auto_repair) { + dout(20) << __func__ << ": auto repair with deep scrubbing" << dendl; + upd_flags.auto_repair = true; + } + upd_flags.calculated_to_deep = true; + dout(20) << fmt::format("{}: final: {}", __func__, upd_flags) << dendl; + return upd_flags; + } + if (allow_shallow_scrub) { + dout(20) << fmt::format("{}: final:{}", __func__, upd_flags) << dendl; + return upd_flags; + } + return std::nullopt; + } + + return std::nullopt; // no scrubbing +} + + +/* + From docs.ceph.com (osd-internals/scrub): + + clang-format off + + Desired no-scrub flags & scrub type interactions: + + Periodic type | none | no-scrub | no-scrub+no-deep | no-deep + ------------------------------------------------------------------------ + ------------------------------------------------------------------------ + periodic | shallow | x | x | shallow + ------------------------------------------------------------------------ + periodic + t.f.deep| deep | deep | x | shallow + ------------------------------------------------------------------------ + initiated | shallow | shallow | shallow | shallow + ------------------------------------------------------------------------ + init. + t.f.deep | deep | deep | shallow | shallow + ------------------------------------------------------------------------ + initiated deep | deep | deep | deep | deep + ------------------------------------------------------------------------ + + "periodic" - if !must_scrub && !must_deep_scrub; + "initiated deep" - if must_scrub && must_deep_scrub; + "initiated" - if must_scrub && !must_deep_scrub; + + clang-format on +*/ +/* + * The returned flags collection (requested_scrub_t) is based on + * m_planned_scrub with the following modifications: + * + * - calculated_to_deep will be set to shallow or deep, depending on the + * scrub type (according to the decision table above); + * - deep_scrub_on_error will be determined; + * - same for auto_repair; + * - time_for_deep will be set to true if the scrub is periodic and the + * time for a deep scrub has been reached (+ some other conditions); + * and + * - need_auto is cleared + */ +std::optional PG::validate_scrub_mode() const +{ + const bool allow_shallow_scrub = + !(get_osdmap()->test_flag(CEPH_OSDMAP_NOSCRUB) || + pool.info.has_flag(pg_pool_t::FLAG_NOSCRUB)); + const bool allow_deep_scrub = + !(get_osdmap()->test_flag(CEPH_OSDMAP_NODEEP_SCRUB) || + pool.info.has_flag(pg_pool_t::FLAG_NODEEP_SCRUB)); + const bool has_deep_errors = (info.stats.stats.sum.num_deep_scrub_errors > 0); + const bool try_to_auto_repair = (cct->_conf->osd_scrub_auto_repair && + get_pgbackend()->auto_repair_supported()); + + dout(10) << __func__ << " pg: " << info.pgid + << " allow: " << allow_shallow_scrub << "/" << allow_deep_scrub + << " deep errs: " << has_deep_errors + << " auto-repair: " << try_to_auto_repair << " (" + << cct->_conf->osd_scrub_auto_repair << ")" << dendl; + + // scrubbing while recovering? + const bool prevented_by_recovery = + osd->is_recovery_active() && !cct->_conf->osd_scrub_during_recovery && + (!cct->_conf->osd_repair_during_recovery || !m_planned_scrub.must_repair); + + if (prevented_by_recovery) { + dout(20) << __func__ << ": scrubbing prevented during recovery" << dendl; + return std::nullopt; + } + + const bool time_for_deep = is_time_for_deep(allow_deep_scrub, + allow_shallow_scrub, + has_deep_errors, + m_planned_scrub); + std::optional upd_flags; + + if (m_planned_scrub.must_scrub) { + upd_flags = validate_initiated_scrub(allow_deep_scrub, + try_to_auto_repair, + time_for_deep, + has_deep_errors, + m_planned_scrub); + } else { + ceph_assert(!m_planned_scrub.must_deep_scrub); + upd_flags = validate_periodic_mode(allow_deep_scrub, + try_to_auto_repair, + allow_shallow_scrub, + time_for_deep, + has_deep_errors, + m_planned_scrub); + if (!upd_flags) { + dout(20) << __func__ << ": no periodic scrubs allowed" << dendl; + return std::nullopt; + } + } + + dout(10) << fmt::format("{}: next scrub flags: {}", __func__, *upd_flags) + << dendl; + upd_flags->need_auto = false; + return upd_flags; +} + + // ///////////////////// preemption_data_t //////////////////////////////////