From 7c02d0e03fe95683be574c8d2e090a5cadec4af4 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Tue, 21 Feb 2023 14:29:01 -0800 Subject: [PATCH] osd/scrubber: use schedule_timer_event_after for scrub sleep Signed-off-by: Samuel Just (cherry picked from commit c82ddeb529a98fcb52c0cd9fb9a8b5b573839901) Signed-off-by: Ronen Friedman --- src/osd/scrubber/pg_scrubber.cc | 68 +++++--------------------- src/osd/scrubber/pg_scrubber.h | 11 ++--- src/osd/scrubber/scrub_machine.cc | 26 +++++++++- src/osd/scrubber/scrub_machine.h | 18 ++++++- src/osd/scrubber/scrub_machine_lstnr.h | 10 ++-- 5 files changed, 61 insertions(+), 72 deletions(-) diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index 2b79f1bb99b..19f04ad3922 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -877,62 +877,6 @@ bool PgScrubber::range_intersects_scrub(const hobject_t& start, return (start < m_max_end && end >= m_start); } -/** - * if we are required to sleep: - * arrange a callback sometimes later. - * be sure to be able to identify a stale callback. - * Otherwise: perform a requeue (i.e. - rescheduling thru the OSD queue) - * anyway. - */ -void PgScrubber::add_delayed_scheduling() -{ - m_end = m_start; // not blocking any range now - - milliseconds sleep_time{0ms}; - sleep_time = m_osds->get_scrub_services().scrub_sleep_time( - m_flags.required); - dout(15) << __func__ << " sleep: " << sleep_time.count() << "ms." << dendl; - - if (sleep_time.count()) { - // schedule a transition for some 'sleep_time' ms in the future - - m_sleep_started_at = ceph_clock_now(); - - // the following log line is used by osd-scrub-test.sh - dout(20) << __func__ << " scrub state is PendingTimer, sleeping" << dendl; - - // the 'delayer' for crimson is different. Will be factored out. - - spg_t pgid = m_pg->get_pgid(); - auto callbk = new LambdaContext([osds = m_osds, pgid, scrbr = this]( - [[maybe_unused]] int r) mutable { - PGRef pg = osds->osd->lookup_lock_pg(pgid); - if (!pg) { - lgeneric_subdout(g_ceph_context, osd, 10) - << "scrub_requeue_callback: Could not find " - << "PG " << pgid << " can't complete scrub requeue after sleep" - << dendl; - return; - } - lgeneric_dout(scrbr->get_pg_cct(), 7) - << "scrub_requeue_callback: slept for " - << ceph_clock_now() - scrbr->m_sleep_started_at << ", re-queuing scrub" - << dendl; - - scrbr->m_sleep_started_at = utime_t{}; - osds->queue_for_scrub_resched(&(*pg), Scrub::scrub_prio_t::low_priority); - pg->unlock(); - }); - - std::lock_guard l(m_osds->sleep_lock); - m_osds->sleep_timer.add_event_after(sleep_time.count() / 1000.0f, callbk); - - } else { - // just a requeue - m_osds->queue_for_scrub_resched(m_pg, Scrub::scrub_prio_t::high_priority); - } -} - eversion_t PgScrubber::search_log_for_updates() const { auto& projected = m_pg->projected_log.log; @@ -2410,6 +2354,17 @@ void PgScrubber::replica_handling_done() reset_internal_state(); } +std::chrono::milliseconds PgScrubber::get_scrub_sleep_time() const +{ + return m_osds->get_scrub_services().scrub_sleep_time( + m_flags.required); +} + +void PgScrubber::queue_for_scrub_resched(Scrub::scrub_prio_t prio) +{ + m_osds->queue_for_scrub_resched(m_pg, prio); +} + /* * note: performs run_callbacks() * note: reservations-related variables are not reset here @@ -2435,7 +2390,6 @@ void PgScrubber::reset_internal_state() m_primary_scrubmap_pos.reset(); replica_scrubmap = ScrubMap{}; replica_scrubmap_pos.reset(); - m_sleep_started_at = utime_t{}; m_active = false; clear_queued_or_active(); diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h index 55bb355f845..e052c916e72 100644 --- a/src/osd/scrubber/pg_scrubber.h +++ b/src/osd/scrubber/pg_scrubber.h @@ -537,11 +537,9 @@ class PgScrubber : public ScrubPgIF, /// services (thus can be called from FSM reactions) void clear_pgscrub_state() final; - /* - * Send an 'InternalSchedScrub' FSM event either immediately, or - if - * 'm_need_sleep' is asserted - after a configuration-dependent timeout. - */ - void add_delayed_scheduling() final; + + std::chrono::milliseconds get_scrub_sleep_time() const final; + void queue_for_scrub_resched(Scrub::scrub_prio_t prio) final; void get_replicas_maps(bool replica_can_preempt) final; @@ -733,9 +731,6 @@ class PgScrubber : public ScrubPgIF, epoch_t m_last_aborted{}; // last time we've noticed a request to abort - utime_t m_sleep_started_at; - - // 'optional', as 'ReplicaReservations' & 'LocalReservation' are // 'RAII-designed' to guarantee un-reserving when deleted. std::optional m_reservations; diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc index 56b06077999..33c1d97a840 100644 --- a/src/osd/scrubber/scrub_machine.cc +++ b/src/osd/scrubber/scrub_machine.cc @@ -288,7 +288,31 @@ PendingTimer::PendingTimer(my_context ctx) dout(10) << "-- state -->> Act/PendingTimer" << dendl; DECLARE_LOCALS; // 'scrbr' & 'pg_id' aliases - scrbr->add_delayed_scheduling(); + auto sleep_time = scrbr->get_scrub_sleep_time(); + if (sleep_time.count()) { + // the following log line is used by osd-scrub-test.sh + dout(20) << __func__ << " scrub state is PendingTimer, sleeping" << dendl; + + dout(20) << "PgScrubber: " << scrbr->get_spgid() + << " sleeping for " << sleep_time << dendl; + m_sleep_timer = machine.schedule_timer_event_after( + sleep_time); + } else { + scrbr->queue_for_scrub_resched(Scrub::scrub_prio_t::high_priority); + } +} + +sc::result PendingTimer::react(const SleepComplete&) +{ + DECLARE_LOCALS; // 'scrbr' & 'pg_id' aliases + dout(10) << "PendingTimer::react(const SleepComplete&)" << dendl; + + auto slept_for = ceph::coarse_real_clock::now() - entered_at; + dout(20) << "PgScrubber: " << scrbr->get_spgid() + << " slept for " << slept_for << dendl; + + scrbr->queue_for_scrub_resched(Scrub::scrub_prio_t::low_priority); + return discard_event(); } // ----------------------- NewChunk ----------------------------------- diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h index 1d999c2cabe..562e5dcdff4 100644 --- a/src/osd/scrubber/scrub_machine.h +++ b/src/osd/scrubber/scrub_machine.h @@ -82,6 +82,8 @@ MEV(InternalSchedScrub) MEV(RangeBlockedAlarm) +MEV(SleepComplete) + MEV(SelectedChunkFree) MEV(ChunkIsBusy) @@ -382,11 +384,25 @@ struct RangeBlocked : sc::state, NamedSimply { sc::result react(const RangeBlockedAlarm &); }; +/** + * PendingTimer + * + * Represents period between chunks. Waits get_scrub_sleep_time() (if non-zero) + * by scheduling a SleepComplete event and then queues an InternalSchedScrub + * to start the next chunk. + */ struct PendingTimer : sc::state, NamedSimply { explicit PendingTimer(my_context ctx); - using reactions = mpl::list>; + using reactions = mpl::list< + sc::transition, + sc::custom_reaction>; + + ceph::coarse_real_clock::time_point entered_at = + ceph::coarse_real_clock::now(); + ScrubMachine::timer_event_token_t m_sleep_timer; + sc::result react(const SleepComplete&); }; struct NewChunk : sc::state, NamedSimply { diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h index 43154f7b61e..15e3c89c89f 100644 --- a/src/osd/scrubber/scrub_machine_lstnr.h +++ b/src/osd/scrubber/scrub_machine_lstnr.h @@ -118,11 +118,11 @@ struct ScrubMachineListener { /// services (thus can be called from FSM reactions) virtual void clear_pgscrub_state() = 0; - /* - * Send an 'InternalSchedScrub' FSM event either immediately, or - if - * 'm_need_sleep' is asserted - after a configuration-dependent timeout. - */ - virtual void add_delayed_scheduling() = 0; + /// Get time to sleep before next scrub + virtual std::chrono::milliseconds get_scrub_sleep_time() const = 0; + + /// Queues InternalSchedScrub for later + virtual void queue_for_scrub_resched(Scrub::scrub_prio_t prio) = 0; /** * Ask all replicas for their scrub maps for the current chunk. -- 2.39.5