From: Ronen Friedman Date: Sun, 1 Sep 2024 07:07:47 +0000 (-0500) Subject: osd/scrub: add configuration parameters to control length of delay X-Git-Tag: testing/wip-pdonnell-testing-20240916.200549-debug~66^2~4 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=706914100325495a50a6335b2e201d762ecfb9f5;p=ceph-ci.git osd/scrub: add configuration parameters to control length of delay to apply to a scrub target following a scrub failure Signed-off-by: Ronen Friedman --- diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in index 268a89154de..946d1bfa1e9 100644 --- a/src/common/options/osd.yaml.in +++ b/src/common/options/osd.yaml.in @@ -520,6 +520,46 @@ options: stats (inc. scrub/block duration) every this many seconds. default: 120 with_legacy: false +- name: osd_scrub_retry_delay + type: int + level: advanced + desc: Period (in seconds) before retrying a PG that has failed a prior scrub. + long_desc: Minimum delay after a failed attempt to scrub a PG. The delay is + either applied to one of the scheduled scrubs for the PG (the next shallow + scrub or the next deep scrub), or to both. + This is a default value, used when the cause of the delay does not have an + associated configuration option. See the 'see also' for the configuration + options for some delay reasons that have their own configuration. + default: 30 + min: 1 + see_also: + - osd_scrub_retry_pg_state + - osd_scrub_retry_after_noscrub + with_legacy: false +- name: osd_scrub_retry_after_noscrub + type: int + level: advanced + desc: Period (in seconds) before retrying to scrub a PG at a specific level + after detecting a no-scrub or no-deep-scrub flag + long_desc: Minimum delay after a failed attempt to scrub a PG at a level + (shallow or deep) that is disabled by cluster or pool no-scrub or no-deep-scrub + flags. + default: 60 + min: 1 + see_also: + - osd_scrub_retry_delay + with_legacy: false +- name: osd_scrub_retry_pg_state + type: int + level: advanced + desc: Period (in seconds) before retrying to scrub a previously inactive/not-clean PG + long_desc: Minimum delay after a failed attempt to scrub a PG that is not + active and clean. + default: 60 + min: 1 + see_also: + - osd_scrub_retry_delay + with_legacy: false - name: osd_scrub_disable_reservation_queuing type: bool level: advanced diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index 0e4253b339a..0e044810fcd 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -2220,7 +2220,7 @@ void PgScrubber::on_mid_scrub_abort(Scrub::delay_cause_t issue) // that made any of the targets into a high-priority one. All that's left: // delay the specific target that was aborted. - auto& trgt = m_scrub_job->delay_on_failure(aborted_target.level(), 5s, issue, + auto& trgt = m_scrub_job->delay_on_failure(aborted_target.level(), issue, scrub_clock_now); /// \todo complete the merging of the deadline & target for non-hp targets @@ -2251,8 +2251,7 @@ void PgScrubber::requeue_penalized( return; } /// \todo fix the 5s' to use a cause-specific delay parameter - auto& trgt = - m_scrub_job->delay_on_failure(s_or_d, 5s, cause, scrub_clock_now); + auto& trgt = m_scrub_job->delay_on_failure(s_or_d, cause, scrub_clock_now); ceph_assert(!trgt.queued); m_osds->get_scrub_services().enqueue_target(trgt); trgt.queued = true; @@ -2274,7 +2273,7 @@ void PgScrubber::requeue_penalized( m_osds->get_scrub_services().dequeue_target(m_pg_id, sister_level); trgt2.queued = false; } - m_scrub_job->delay_on_failure(sister_level, 5s, cause, scrub_clock_now); + m_scrub_job->delay_on_failure(sister_level, cause, scrub_clock_now); m_osds->get_scrub_services().enqueue_target(trgt2); trgt2.queued = true; } diff --git a/src/osd/scrubber/scrub_job.cc b/src/osd/scrubber/scrub_job.cc index ee33ee06706..dd9f8b56de0 100644 --- a/src/osd/scrubber/scrub_job.cc +++ b/src/osd/scrubber/scrub_job.cc @@ -12,6 +12,7 @@ using sched_conf_t = Scrub::sched_conf_t; using scrub_schedule_t = Scrub::scrub_schedule_t; using ScrubJob = Scrub::ScrubJob; using delay_ready_t = Scrub::delay_ready_t; +using namespace std::chrono; namespace { utime_t add_double(utime_t t, double d) @@ -300,16 +301,35 @@ void ScrubJob::adjust_deep_schedule( SchedTarget& ScrubJob::delay_on_failure( scrub_level_t level, - std::chrono::seconds delay, Scrub::delay_cause_t delay_cause, utime_t scrub_clock_now) { + seconds delay = seconds(cct->_conf.get_val("osd_scrub_retry_delay")); + switch (delay_cause) { + case delay_cause_t::flags: + delay = + seconds(cct->_conf.get_val("osd_scrub_retry_after_noscrub")); + break; + case delay_cause_t::pg_state: + delay = seconds(cct->_conf.get_val("osd_scrub_retry_pg_state")); + break; + case delay_cause_t::local_resources: + default: + // for all other possible delay causes: use the default delay + break; + } + auto& delayed_target = (level == scrub_level_t::deep) ? deep_target : shallow_target; delayed_target.sched_info.schedule.not_before = std::max(scrub_clock_now, delayed_target.sched_info.schedule.not_before) + utime_t{delay}; delayed_target.sched_info.last_issue = delay_cause; + dout(20) << fmt::format( + "delayed {}scrub due to {} for {}s. Updated: {}", + (level == scrub_level_t::deep ? "deep " : ""), delay_cause, + delay.count(), delayed_target) + << dendl; return delayed_target; } diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h index 98a3e101f9b..b037084db6b 100644 --- a/src/osd/scrubber/scrub_job.h +++ b/src/osd/scrubber/scrub_job.h @@ -239,14 +239,14 @@ class ScrubJob { /** * For the level specified, set the 'not-before' time to 'now+delay', - * so that this scrub target - * would not be retried before 'delay' seconds have passed. + * so that this scrub target would not be retried before the required + * delay seconds have passed. + * The delay is determined based on the 'cause' parameter. * The 'last_issue' is updated to the cause of the delay. * \returns a reference to the target that was modified. */ [[maybe_unused]] SchedTarget& delay_on_failure( scrub_level_t level, - std::chrono::seconds delay, delay_cause_t delay_cause, utime_t scrub_clock_now);