osd: scrubber: formalizing the scrub preemption states

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 3c11c88b348e321f791cda42d2d41e620b81fbaa..69f631394ef98895a2cb152752991925349e9fc6 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -66,6 +66,7 @@ struct OpRequest;
 typedef OpRequest::Ref OpRequestRef;
 class MOSDPGLog;
 class DynamicPerfStats;
+class PgScrubber;
 
 namespace Scrub {
   class Store;
@@ -168,6 +169,7 @@ class PGRecoveryStats {
 class PG : public DoutPrefixProvider, public PeeringState::PeeringListener {
   friend struct NamedState;
   friend class PeeringState;
+  friend class PgScrubber;
   friend class Scrub::ReplicaReservations;
   friend class Scrub::LocalReservation;  // dout()-only friendship
   friend class Scrub::ReservedByRemotePrimary;  //  dout()-only friendship

diff --git a/src/osd/pg_scrubber.cc b/src/osd/pg_scrubber.cc
index 763e7769147e67b5dafc62a604c96fcf5fb67f97..b6af7e07fc71c2d9eae2710478a695f742eac0f5 100644 (file)
--- a/src/osd/pg_scrubber.cc
+++ b/src/osd/pg_scrubber.cc
@@ -58,6 +58,26 @@ ostream& operator<<(ostream& out, const requested_scrub_t& sf)
   return out;
 }
 
+// ///////////////////// preemption_data_t //////////////////////////////////
+
+PgScrubber::preemption_data_t::preemption_data_t(PG* pg) : m_pg{pg}
+{
+  m_left = static_cast<int>(
+    m_pg->get_cct()->_conf.get_val<uint64_t>("osd_scrub_max_preemptions"));
+}
+
+void PgScrubber::preemption_data_t::reset()
+{
+  std::lock_guard<std::mutex> lk{m_preemption_lock};
+
+  m_preemptable = false;
+  m_preempted = false;
+  m_left =
+    static_cast<int>(m_pg->cct->_conf.get_val<uint64_t>("osd_scrub_max_preemptions"));
+  m_size_divisor = 1;
+}
+
+
 // ///////////////////// ReplicaReservations //////////////////////////////////
 namespace Scrub {
 
@@ -66,7 +86,7 @@ void ReplicaReservations::release_replica(pg_shard_t peer, epoch_t epoch)
   dout(15) << __func__ << " <ReplicaReservations> release-> " << peer << dendl;
 
   auto m = new MOSDScrubReserve(spg_t(m_pg->info.pgid.pgid, peer.shard), epoch,
-                                   MOSDScrubReserve::RELEASE, m_pg->pg_whoami);
+                               MOSDScrubReserve::RELEASE, m_pg->pg_whoami);
   m_osds->send_message_osd_cluster(peer.osd, m, epoch);
 }
 
@@ -89,7 +109,7 @@ ReplicaReservations::ReplicaReservations(PG* pg, pg_shard_t whoami)
       if (p == whoami)
        continue;
       auto m = new MOSDScrubReserve(spg_t(m_pg->info.pgid.pgid, p.shard), epoch,
-                                       MOSDScrubReserve::REQUEST, m_pg->pg_whoami);
+                                   MOSDScrubReserve::REQUEST, m_pg->pg_whoami);
       m_osds->send_message_osd_cluster(p.osd, m, epoch);
       m_waited_for_peers.push_back(p);
       dout(10) << __func__ << " <ReplicaReservations> reserve<-> " << p.osd << dendl;

diff --git a/src/osd/pg_scrubber.h b/src/osd/pg_scrubber.h
index 7d5ecfd6148a7cb48124bd7d33228659f7708d73..760c34310c6966aadfdc4920c0450522a25566d1 100644 (file)
--- a/src/osd/pg_scrubber.h
+++ b/src/osd/pg_scrubber.h
@@ -131,5 +131,96 @@ class MapsCollectionStatus {
   friend ostream& operator<<(ostream& out, const MapsCollectionStatus& sf);
 };
 
-
 }  // namespace Scrub
+
+// an almost-empty PgScrubber for this commit:
+class PgScrubber : public ScrubPgIF, public ScrubMachineListener {
+
+  /**
+   * the 'preemption' "state-machine".
+   * Note: I was considering an orthogonal sub-machine implementation, but as
+   * the state diagram is extremely simple, the added complexity wasn't justified.
+   */
+  class preemption_data_t : public Scrub::preemption_t {
+   public:
+    preemption_data_t(PG* pg); // the PG access is used for conf access (and logs)
+
+    [[nodiscard]] bool is_preemptable() const final { return m_preemptable; }
+
+    bool do_preempt() final
+    {
+      if (m_preempted || !m_preemptable)
+       return false;
+
+      std::lock_guard<std::mutex> lk{m_preemption_lock};
+      if (!m_preemptable)
+       return false;
+
+      m_preempted = true;
+      return true;
+    }
+
+    /// same as 'do_preempt()' but w/o checks (as once a replica
+    /// was preempted, we cannot continue)
+    void replica_preempted() { m_preempted = true; }
+
+    void enable_preemption()
+    {
+      std::lock_guard<std::mutex> lk{m_preemption_lock};
+      if (are_preemptions_left() && !m_preempted) {
+       m_preemptable = true;
+      }
+    }
+
+    /// used by a replica to set preemptability state according to the Primary's request
+    void force_preemptability(bool is_allowed)
+    {
+      // note: no need to lock for a replica
+      m_preempted = false;
+      m_preemptable = is_allowed;
+    }
+
+    bool disable_and_test() final
+    {
+      std::lock_guard<std::mutex> lk{m_preemption_lock};
+      m_preemptable = false;
+      return m_preempted;
+    }
+
+    [[nodiscard]] bool was_preempted() const { return m_preempted; }
+
+    [[nodiscard]] size_t chunk_divisor() const { return m_size_divisor; }
+
+    void reset();
+
+    void adjust_parameters() final
+    {
+      std::lock_guard<std::mutex> lk{m_preemption_lock};
+
+      if (m_preempted) {
+       m_preempted = false;
+       m_preemptable = adjust_left();
+      } else {
+       m_preemptable = are_preemptions_left();
+      }
+    }
+
+   private:
+    PG* m_pg;
+    mutable std::mutex m_preemption_lock;
+    bool m_preemptable{false};
+    bool m_preempted{false};
+    int m_left;
+    size_t m_size_divisor{1};
+    bool are_preemptions_left() const { return m_left > 0; }
+
+    bool adjust_left()
+    {
+      if (m_left > 0) {
+       --m_left;
+       m_size_divisor *= 2;
+      }
+      return m_left > 0;
+    }
+  };
+};