{
auto& scrub_scheduler = service.get_scrub_services();
+ if (auto blocked_pgs = scrub_scheduler.get_blocked_pgs_count();
+ blocked_pgs > 0) {
+ // some PGs managed by this OSD were blocked by a locked object during
+ // scrub. This means we might not have the resources needed to scrub now.
+ dout(10)
+ << fmt::format(
+ "{}: PGs are blocked while scrubbing due to locked objects ({} PGs)",
+ __func__,
+ blocked_pgs)
+ << dendl;
+ }
+
// fail fast if no resources are available
if (!scrub_scheduler.can_inc_scrubs()) {
dout(20) << __func__ << ": OSD cannot inc scrubs" << dendl;
std::string pg_stat_t::dump_scrub_schedule() const
{
if (scrub_sched_status.m_is_active) {
- return fmt::format(
- "{}scrubbing for {}s",
- ((scrub_sched_status.m_is_deep == scrub_level_t::deep) ? "deep " : ""),
- scrub_sched_status.m_duration_seconds);
+ // are we blocked (in fact, stuck) on some locked object?
+ if (scrub_sched_status.m_sched_status == pg_scrub_sched_status_t::blocked) {
+ return fmt::format(
+ "Blocked! locked objects (for {}s)",
+ scrub_sched_status.m_duration_seconds);
+ } else {
+ return fmt::format(
+ "{}scrubbing for {}s",
+ ((scrub_sched_status.m_is_deep == scrub_level_t::deep) ? "deep " : ""),
+ scrub_sched_status.m_duration_seconds);
+ }
}
switch (scrub_sched_status.m_sched_status) {
case pg_scrub_sched_status_t::unknown:
not_queued, ///< not in the OSD's scrub queue. Probably not active.
active, ///< scrubbing
scheduled, ///< scheduled for a scrub at an already determined time
- queued ///< queued to be scrubbed
+ queued, ///< queued to be scrubbed
+ blocked ///< blocked waiting for objects to be unlocked
};
struct pg_scrubbing_status_t {
f->dump_int("scrubs_remote", scrubs_remote);
f->dump_int("osd_max_scrubs", conf()->osd_max_scrubs);
}
+
+void ScrubQueue::clear_pg_scrub_blocked(spg_t blocked_pg)
+{
+ dout(5) << fmt::format(": pg {} is unblocked", blocked_pg) << dendl;
+ --blocked_scrubs_cnt;
+ ceph_assert(blocked_scrubs_cnt >= 0);
+}
+
+void ScrubQueue::mark_pg_scrub_blocked(spg_t blocked_pg)
+{
+ dout(5) << fmt::format(": pg {} is blocked on an object", blocked_pg)
+ << dendl;
+ ++blocked_scrubs_cnt;
+}
+
+int ScrubQueue::get_blocked_pgs_count() const
+{
+ return blocked_scrubs_cnt;
+}
*/
std::atomic_bool updated{false};
+ /**
+ * the scrubber is waiting for locked objects to be unlocked.
+ * Set after a grace period has passed.
+ */
+ bool blocked{false};
+ utime_t blocked_since{};
+
utime_t penalty_timeout{0, 0};
CephContext* cct;
void dec_scrubs_remote();
void dump_scrub_reservations(ceph::Formatter* f) const;
+ /// counting the number of PGs stuck while scrubbing, waiting for objects
+ void mark_pg_scrub_blocked(spg_t blocked_pg);
+ void clear_pg_scrub_blocked(spg_t blocked_pg);
+ int get_blocked_pgs_count() const;
+
/**
* Pacing the scrub operation by inserting delays (mostly between chunks)
*
int scrubs_local{0};
int scrubs_remote{0};
+ /**
+ * The scrubbing of PGs might be delayed if the scrubbed chunk of objects is
+ * locked by some other operation. A bug might cause this to be an infinite
+ * delay. If that happens, the OSDs "scrub resources" (i.e. the
+ * counters that limit the number of concurrent scrub operations) might
+ * be exhausted.
+ * We do issue a cluster-log warning in such occasions, but that message is
+ * easy to miss. The 'some pg is blocked' global flag is used to note the
+ * existence of such a situation in the scrub-queue log messages.
+ */
+ std::atomic_int_fast16_t blocked_scrubs_cnt{0};
+
std::atomic_bool a_pg_is_reserving{false};
[[nodiscard]] bool scrub_load_below_threshold() const;
m_pg->info,
m_pg->get_pgpool().info.opts);
m_osds->get_scrub_services().update_job(m_scrub_job, suggested);
+ m_pg->publish_stats_to_osd();
}
dout(15) << __func__ << " done " << registration_state() << dendl;
<< dendl;
update_scrub_job(req_flags);
- m_pg->publish_stats_to_osd();
}
Scrub::BlockedRangeWarning PgScrubber::acquire_blocked_alarm()
{
+ ceph::timespan grace_period{m_debug_blockrange?4s:120s};
return std::make_unique<blocked_range_t>(m_osds,
- ceph::timespan{300s},
+ grace_period,
+ *this,
m_pg_id);
}
return m_queued_or_active;
}
+void PgScrubber::set_scrub_blocked(utime_t since)
+{
+ ceph_assert(!m_scrub_job->blocked);
+ // we are called from a time-triggered lambda,
+ // thus - not under PG-lock
+ PGRef pg = m_osds->osd->lookup_lock_pg(m_pg_id);
+ m_osds->get_scrub_services().mark_pg_scrub_blocked(m_pg_id);
+ m_scrub_job->blocked_since = since;
+ m_scrub_job->blocked = true;
+ m_pg->publish_stats_to_osd();
+ pg->unlock();
+}
+
+void PgScrubber::clear_scrub_blocked()
+{
+ ceph_assert(m_scrub_job->blocked);
+ m_osds->get_scrub_services().clear_pg_scrub_blocked(m_pg_id);
+ m_scrub_job->blocked = false;
+ m_pg->publish_stats_to_osd();
+}
+
/*
* note: only called for the Primary.
*/
}
f->close_section();
}
- f->dump_string("schedule", "scrubbing");
+ if (m_scrub_job->blocked) {
+ f->dump_string("schedule", "blocked");
+ } else {
+ f->dump_string("schedule", "scrubbing");
+ }
}
pg_scrubbing_status_t PgScrubber::get_schedule() const
{
- dout(25) << __func__ << dendl;
-
if (!m_scrub_job) {
return pg_scrubbing_status_t{};
}
+ dout(25) << fmt::format("{}: active:{} blocked:{}",
+ __func__,
+ m_active,
+ m_scrub_job->blocked)
+ << dendl;
+
auto now_is = ceph_clock_now();
if (m_active) {
// report current scrub info, including updated duration
- int32_t duration = (utime_t{now_is} - scrub_begin_stamp).sec();
- return pg_scrubbing_status_t{
- utime_t{},
- duration,
- pg_scrub_sched_status_t::active,
- true, // active
- (m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow),
- false /* is periodic? unknown, actually */};
+ if (m_scrub_job->blocked) {
+ // a bug. An object is held locked.
+ int32_t blocked_for =
+ (utime_t{now_is} - m_scrub_job->blocked_since).sec();
+ return pg_scrubbing_status_t{
+ utime_t{},
+ blocked_for,
+ pg_scrub_sched_status_t::blocked,
+ true, // active
+ (m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow),
+ false};
+
+ } else {
+ int32_t duration = (utime_t{now_is} - scrub_begin_stamp).sec();
+ return pg_scrubbing_status_t{
+ utime_t{},
+ duration,
+ pg_scrub_sched_status_t::active,
+ true, // active
+ (m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow),
+ false /* is periodic? unknown, actually */};
+ }
}
if (m_scrub_job->state != ScrubQueue::qu_state_t::registered) {
return pg_scrubbing_status_t{utime_t{},
blocked_range_t::blocked_range_t(OSDService* osds,
ceph::timespan waittime,
+ ScrubMachineListener& scrubber,
spg_t pg_id)
: m_osds{osds}
+ , m_scrubber{scrubber}
+ , m_pgid{pg_id}
{
auto now_is = std::chrono::system_clock::now();
- m_callbk = new LambdaContext([now_is, pg_id, osds]([[maybe_unused]] int r) {
+ m_callbk = new LambdaContext([this, now_is]([[maybe_unused]] int r) {
std::time_t now_c = std::chrono::system_clock::to_time_t(now_is);
char buf[50];
strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", std::localtime(&now_c));
lgeneric_subdout(g_ceph_context, osd, 10)
- << "PgScrubber: " << pg_id << " blocked on an object for too long (since "
- << buf << ")" << dendl;
- osds->clog->warn() << "osd." << osds->whoami << " PgScrubber: " << pg_id
- << " blocked on an object for too long (since " << buf
- << ")";
+ << "PgScrubber: " << m_pgid
+ << " blocked on an object for too long (since " << buf << ")" << dendl;
+ m_osds->clog->warn() << "osd." << m_osds->whoami
+ << " PgScrubber: " << m_pgid
+ << " blocked on an object for too long (since " << buf
+ << ")";
+
+ m_warning_issued = true;
+ m_scrubber.set_scrub_blocked(utime_t{now_c,0});
return;
});
blocked_range_t::~blocked_range_t()
{
+ if (m_warning_issued) {
+ m_scrubber.clear_scrub_blocked();
+ }
std::lock_guard l(m_osds->sleep_lock);
m_osds->sleep_timer.cancel_event(m_callbk);
}
void select_range_n_notify() final;
Scrub::BlockedRangeWarning acquire_blocked_alarm() final;
+ void set_scrub_blocked(utime_t since) final;
+ void clear_scrub_blocked() final;
+
/// walk the log to find the latest update that affects our chunk
eversion_t search_log_for_updates() const final;
#include "include/Context.h"
#include "osd/osd_types.h"
+struct ScrubMachineListener;
+
namespace Scrub {
enum class PreemptionNoted { no_preemption, preempted };
/// an aux used when blocking on a busy object.
/// Issues a log warning if still blocked after 'waittime'.
struct blocked_range_t {
- blocked_range_t(OSDService* osds, ceph::timespan waittime, spg_t pg_id);
+ blocked_range_t(OSDService* osds,
+ ceph::timespan waittime,
+ ScrubMachineListener& scrubber,
+ spg_t pg_id);
~blocked_range_t();
OSDService* m_osds;
+ ScrubMachineListener& m_scrubber;
+
+ /// used to identify ourselves to the PG, when no longer blocked
+ spg_t m_pgid;
Context* m_callbk;
+
+ // once timed-out, we flag the OSD's scrub-queue as having
+ // a problem. 'm_warning_issued' signals the need to clear
+ // that OSD-wide flag.
+ bool m_warning_issued{false};
};
using BlockedRangeWarning = std::unique_ptr<blocked_range_t>;
virtual void set_queued_or_active() = 0;
virtual void clear_queued_or_active() = 0;
+ /**
+ * Our scrubbing is blocked, waiting for an excessive length of time for
+ * our target chunk to be unlocked. We will set the corresponding flags,
+ * both in the OSD_wide scrub-queue object, and in our own scrub-job object.
+ * Both flags are used to report the unhealthy state in the log and in
+ * response to scrub-queue queries.
+ */
+ virtual void set_scrub_blocked(utime_t since) = 0;
+ virtual void clear_scrub_blocked() = 0;
+
/**
* the FSM interface into the "are we waiting for maps, either our own or from
* replicas" state.
projects / ceph.git / commitdiff