osd/scrub: support an operator-abort command

The new explicit command aborts any ongoing scrub of the target PG,
including operator-initiated scrubs. That additional capability is needed now that
operator-initiated scrubs are no longer blocked by 'no-scrub' settings.

The scenario we are trying to help the operator with is:
- an operator issues a set of operator-initiated scrubs (e.g., via a
  script), then realizes the mistake and wants to abort them all.
The abort command also downgrades the urgency level of the scrub target
(as otherwise the target would immediately restart, against the operator
wishes).

This commit implements the changes to the state machine and to the abort
logic, assuming the operator command was translated into an event.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
(cherry picked from commit d24a02f67a26f1bf8f13efffe2fa213c744f096f)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index b4be85680c84bd9c3e0e655c7d1050c1e0df0d48..320292dda1789269cab3fa5076c16ca937470448 100644 (file)
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -2061,6 +2061,26 @@ void PgScrubber::on_digest_updates()
   }
 }
 
+bool PgScrubber::downgrade_on_operator_abort(
+    Scrub::SchedTarget& targ,
+    utime_t scrub_clock_now)
+{
+  if (targ.urgency() != urgency_t::operator_requested &&
+      targ.urgency() != urgency_t::must_repair) {
+    return false;  // no need to downgrade
+  }
+
+  targ.sched_info.urgency = urgency_t::periodic_regular;
+  targ.sched_info.schedule.scheduled_at = scrub_clock_now;
+  targ.sched_info.schedule.not_before = scrub_clock_now;
+  dout(10)
+      << fmt::format(
+             "{}: removing operator-requested urgency from target. Updated: {}",
+             __func__, targ)
+      << dendl;
+  return true;
+}
+
 
 /**
  * The scrub session was aborted. We are left with two sets of parameters
@@ -2072,6 +2092,10 @@ void PgScrubber::on_digest_updates()
  * have had its priority, flags, or schedule modified in the meantime.
  * And - it does not (at least initially, i.e. immediately after
  * set_op_parameters()), have high priority.
+ *
+ * Updated functionality ('Tentacle', 2025): if the abort cause was an explicit
+ * operator request - make sure we are not left with a high-priority
+ * target (one that would immediately restart, against the operator wishes).
  */
 void PgScrubber::on_mid_scrub_abort(Scrub::delay_cause_t issue)
 {
@@ -2086,38 +2110,54 @@ void PgScrubber::on_mid_scrub_abort(Scrub::delay_cause_t issue)
 
   dout(10) << fmt::format(
                  "{}: executing target: {}. Session flags: {} up-to-date job: "
-                 "{}",
-                 __func__, *m_active_target, m_flags, *m_scrub_job)
+                 "{}. Abort cause: {}",
+                 __func__, *m_active_target, m_flags, *m_scrub_job, issue)
           << dendl;
 
   // copy the aborted target
-  const auto aborted_target = *m_active_target;
+  auto aborted_target = *m_active_target;
   m_active_target.reset();
 
   const auto scrub_clock_now = ceph_clock_now();
   auto& current_targ = m_scrub_job->get_target(aborted_target.level());
   ceph_assert(!current_targ.queued);
 
-  // merge the aborted target with the current one
-  auto& curr_sched = current_targ.sched_info.schedule;
-  auto& abrt_sched = aborted_target.sched_info.schedule;
-
-  current_targ.sched_info.urgency =
-      std::max(current_targ.urgency(), aborted_target.urgency());
-  curr_sched.scheduled_at =
-      std::min(curr_sched.scheduled_at, abrt_sched.scheduled_at);
-  curr_sched.not_before =
-      std::min(curr_sched.not_before, abrt_sched.not_before);
-
-  dout(10) << fmt::format(
-                 "{}: merged target (before delay): {}", __func__,
-                 current_targ)
-          << dendl;
+  // if the abort trigger was an explicit operator abort command, and the
+  // aborted target had operator-initiated urgency:
+  // - do not perform a 'merge' of the aborted target and the 'next'
+  //   target in the scrub-job. Instead - just reinstate the 'next' target.
+  //   (the aborted target has more than its urgency attribute wrong. The
+  //   scheduled-at was also made irrelevant by the original operator
+  //   command that initiated the aborted scrub).
+  bool should_merge = true;
+  if (issue == delay_cause_t::operator_abort) {
+    should_merge = !downgrade_on_operator_abort(aborted_target, scrub_clock_now);
+  }
+
+  if (should_merge) {
+    // the regular case. merge the aborted target with the current one
+    auto& curr_sched = current_targ.sched_info.schedule;
+    auto& abrt_sched = aborted_target.sched_info.schedule;
+
+    current_targ.sched_info.urgency =
+        std::max(current_targ.urgency(), aborted_target.urgency());
+    curr_sched.scheduled_at =
+        std::min(curr_sched.scheduled_at, abrt_sched.scheduled_at);
+    curr_sched.not_before =
+        std::min(curr_sched.not_before, abrt_sched.not_before);
+    dout(10) << fmt::format(
+                   "{}: merged target (before delay): {}", __func__,
+                   current_targ)
+            << dendl;
+  } else {
+    dout(10) << fmt::format(
+                    "{}: aborted oper-urgency target discarded: {}",
+                    __func__, current_targ)
+             << dendl;
+  }
 
   // affect a delay, as there was a failure mid-scrub
   m_scrub_job->delay_on_failure(current_targ.level(), issue, scrub_clock_now);
-
-  // reinstate both targets in the queue
   m_osds->get_scrub_services().enqueue_target(current_targ);
   current_targ.queued = true;
 
@@ -2125,7 +2165,13 @@ void PgScrubber::on_mid_scrub_abort(Scrub::delay_cause_t issue)
   auto& sister = m_scrub_job->get_target(
       aborted_target.level() == scrub_level_t::deep ? scrub_level_t::shallow
                                                    : scrub_level_t::deep);
+  // if 'operator-aborted' - that one should be downgraded, too (the scenario
+  // we are trying to help the operator with: trying to recover from a set of
+  // scrub requests issued by mistake).
   if (!sister.queued) {
+    if (issue == delay_cause_t::operator_abort) {
+      downgrade_on_operator_abort(sister, scrub_clock_now);
+    }
     m_osds->get_scrub_services().enqueue_target(sister);
     sister.queued = true;
   }

diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index a4636692e5d27f83ad3b6525048ab453d64ac52d..f3e2d1c5d2f93bcb063a8e4335b962d62cc9e29b 100644 (file)
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -516,6 +516,17 @@ class PgScrubber : public ScrubPgIF,
 
   void on_mid_scrub_abort(Scrub::delay_cause_t issue) final;
 
+  /**
+   *  an auxiliary used by on_mid_scrub_abort()
+   *  If the target has operator-initiated urgency (either 'must_repair' -
+   *  operator-requested repair or 'operator_requested' - operator-requested
+   *  scrub) - downgrade it to regular periodic.
+   *  \retval true: the urgency was downgraded
+   */
+  bool downgrade_on_operator_abort(
+    Scrub::SchedTarget& targ,
+    utime_t scrub_clock_now);
+
   ScrubMachineListener::MsgAndEpoch prep_replica_map_msg(
     Scrub::PreemptionNoted was_preempted) final;
 

diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 76be5a9f5a0225665e7c68f032412b738e53aee5..6d041d2a94e639f3e228f55195a3cefb65b40779 100644 (file)
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -224,6 +224,15 @@ sc::result Session::react(const IntervalChanged&)
   return transit<NotActive>();
 }
 
+sc::result Session::react(const OperatorAbort&)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "Session::react(const OperatorAbort&)" << dendl;
+  ceph_assert(m_reservations);
+  m_abort_reason = delay_cause_t::operator_abort;
+  return transit<PrimaryIdle>();
+}
+
 std::optional<pg_scrubbing_status_t> Session::get_reservation_status() const
 {
   if (!m_reservations) {

diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index db3905ca478b563765ff5978000244e752164ff5..e8d07af5a956a02dc0b31b57240ae5f0f84f82c6 100644 (file)
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -247,6 +247,12 @@ MEV(IntervalChanged)
  */
 MEV(FullReset)
 
+/**
+ * (Primary only) stops the running scrub. Removes any higher-than-periodic
+ * 'urgency' attributes.
+ */
+MEV(OperatorAbort)
+
 /// finished handling this chunk. Go get the next one
 MEV(NextChunk)
 
@@ -557,10 +563,13 @@ struct Session : sc::state<Session, PrimaryActive, ReservingReplicas>,
   ~Session();
 
   using reactions = mpl::list<sc::transition<FullReset, PrimaryIdle>,
-                              sc::custom_reaction<IntervalChanged>>;
+                              sc::custom_reaction<IntervalChanged>,
+                              sc::custom_reaction<OperatorAbort>>;
 
   sc::result react(const IntervalChanged&);
 
+  sc::result react(const OperatorAbort&);
+
   /// managing the scrub session's reservations (optional, as
   /// it's an RAII wrapper around the state of 'holding reservations')
   std::optional<ReplicaReservations> m_reservations{std::nullopt};

diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 0527addc0f65421a83d5f4a43ffab9f6440168bf..9b2e03ee4688f10bad12c3167e8efb74e46006d8 100644 (file)
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -216,6 +216,7 @@ enum class delay_cause_t {
   aborted,         ///< scrub was aborted w/ unspecified reason
   interval,        ///< the interval had ended mid-scrub
   scrub_params,     ///< the specific scrub type is not allowed
+  operator_abort    ///< operator-requested abort
 };
 }  // namespace Scrub
 
@@ -239,6 +240,7 @@ struct formatter<Scrub::delay_cause_t> : ::fmt::formatter<std::string_view> {
       case aborted:             desc = "aborted"; break;
       case interval:            desc = "interval"; break;
       case scrub_params:        desc = "scrub-mode"; break;
+      case operator_abort:      desc = "operator-abort"; break;
       // better to not have a default case, so that the compiler will warn
     }
     return ::fmt::formatter<string_view>::format(desc, ctx);