osd_recovery_sleep_degraded, osd_recovery_sleep_degraded_ssd, osd_recovery_sleep_degr...

Fixes: https://tracker.ceph.com/issues/67700
Signed-off-by: Md Mahamudur Rahaman Sajib <mahamudur.sajib@croit.io>

diff --git a/doc/rados/configuration/mclock-config-ref.rst b/doc/rados/configuration/mclock-config-ref.rst
index 58de3e54bfef9a0df8a79921a1a1637fbf304f52..c1287a21f1b532d0da667396668a9cfeb043e96d 100644 (file)
--- a/doc/rados/configuration/mclock-config-ref.rst
+++ b/doc/rados/configuration/mclock-config-ref.rst
@@ -292,6 +292,10 @@ sleep options are disabled (set to 0),
 - :confval:`osd_recovery_sleep_hdd`
 - :confval:`osd_recovery_sleep_ssd`
 - :confval:`osd_recovery_sleep_hybrid`
+- :confval:`osd_recovery_sleep_degraded`
+- :confval:`osd_recovery_sleep_degraded_hdd`
+- :confval:`osd_recovery_sleep_degraded_ssd`
+- :confval:`osd_recovery_sleep_degraded_hybrid`
 - :confval:`osd_scrub_sleep`
 - :confval:`osd_delete_sleep`
 - :confval:`osd_delete_sleep_hdd`

diff --git a/doc/rados/configuration/osd-config-ref.rst b/doc/rados/configuration/osd-config-ref.rst
index 23efa797773aab9341ba74031ac0b429b79e517e..df527f99fbd2d0d75ae70f84705bdf99c6fa3557 100644 (file)
--- a/doc/rados/configuration/osd-config-ref.rst
+++ b/doc/rados/configuration/osd-config-ref.rst
@@ -431,6 +431,10 @@ perform well in a degraded state.
 .. confval:: osd_recovery_sleep_hdd
 .. confval:: osd_recovery_sleep_ssd
 .. confval:: osd_recovery_sleep_hybrid
+.. confval:: osd_recovery_sleep_degraded
+.. confval:: osd_recovery_sleep_degraded_hdd
+.. confval:: osd_recovery_sleep_degraded_ssd
+.. confval:: osd_recovery_sleep_degraded_hybrid
 .. confval:: osd_recovery_priority
 
 Tiering

diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in
index 2be1c08f934c82d163d2313e2cae94e7c4382537..4dc60d33ff01d7c9eecf1367ba8542d26383c733 100644 (file)
--- a/src/common/options/osd.yaml.in
+++ b/src/common/options/osd.yaml.in
@@ -140,6 +140,51 @@ options:
   - osd_recovery_sleep
   flags:
   - runtime
+- name: osd_recovery_sleep_degraded
+  type: float
+  level: advanced
+  desc: Time in seconds to sleep before next recovery or backfill op when PGs are degraded.
+    This setting overrides _ssd, _hdd, and _hybrid if non-zero.
+  fmt_desc: Time in seconds to sleep before the next recovery or backfill op when PGs
+    are degraded. Increasing this value will slow down recovery ops while client
+    ops will be less impacted.
+  default: 0
+  flags:
+  - runtime
+- name: osd_recovery_sleep_degraded_hdd
+  type: float
+  level: advanced
+  desc: Time in seconds to sleep before next recovery or backfill op for HDDs
+    when PGs is degraded.
+  fmt_desc: Time in seconds to sleep before next recovery or backfill op
+    for HDDs when PGs are degraded.
+  default: 0.1
+  flags:
+  - runtime
+- name: osd_recovery_sleep_degraded_ssd
+  type: float
+  level: advanced
+  desc: Time in seconds to sleep before next recovery or backfill op for SSDs
+    when PGs are degraded.
+  fmt_desc: Time in seconds to sleep before the next recovery or backfill op
+    for SSDs when PGs are degraded.
+  default: 0
+  see_also:
+  - osd_recovery_sleep_degraded
+  flags:
+  - runtime
+- name: osd_recovery_sleep_degraded_hybrid
+  type: float
+  level: advanced
+  desc: Time in seconds to sleep before next recovery or backfill op when PGs
+    are degraded and data is on HDD and journal is on SSD
+  fmt_desc: Time in seconds to sleep before the next recovery or backfill op when
+    PGs are degraded and OSD data is on HDD and OSD journal / WAL+DB is on SSD.
+  default: 0.025
+  see_also:
+  - osd_recovery_sleep_degraded
+  flags:
+  - runtime
 - name: osd_snap_trim_sleep
   type: float
   level: advanced

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index af3bd8fe0e8cf89f72ddbb71b17581a43fb507b1..54167bc1df33fc95c363212330ede4bd5c816062 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3681,6 +3681,21 @@ float OSD::get_osd_recovery_sleep()
     return cct->_conf->osd_recovery_sleep_hdd;
 }
 
+float OSD::get_osd_recovery_sleep_degraded() {
+  float osd_recovery_sleep_degraded =
+    cct->_conf.get_val<double>("osd_recovery_sleep_degraded");
+  if (osd_recovery_sleep_degraded > 0) {
+    return osd_recovery_sleep_degraded;
+  }
+  if (!store_is_rotational && !journal_is_rotational) {
+    return cct->_conf.get_val<double>("osd_recovery_sleep_degraded_ssd");
+  } else if (store_is_rotational && !journal_is_rotational) {
+    return cct->_conf.get_val<double>("osd_recovery_sleep_degraded_hybrid");
+  } else {
+    return cct->_conf.get_val<double>("osd_recovery_sleep_degraded_hdd");
+  }
+}
+
 float OSD::get_osd_delete_sleep()
 {
   float osd_delete_sleep = cct->_conf.get_val<double>("osd_delete_sleep");
@@ -9703,9 +9718,12 @@ void OSD::do_recovery(
    * ops are scheduled after osd_recovery_sleep amount of time from the previous
    * recovery event's schedule time. This is done by adding a
    * recovery_requeue_callback event, which re-queues the recovery op using
-   * queue_recovery_after_sleep.
+   * queue_recovery_after_sleep. (osd_recovery_sleep_degraded will be
+   * used instead of osd_recovery_sleep when pg is degraded)
    */
-  float recovery_sleep = get_osd_recovery_sleep();
+  float recovery_sleep = pg->is_degraded() 
+                        ? get_osd_recovery_sleep_degraded() 
+                        : get_osd_recovery_sleep();
   {
     std::lock_guard l(service.sleep_lock);
     if (recovery_sleep > 0 && service.recovery_needs_sleep) {
@@ -10014,6 +10032,10 @@ std::vector<std::string> OSD::get_tracked_keys() const noexcept
     "osd_recovery_sleep_hdd"s,
     "osd_recovery_sleep_ssd"s,
     "osd_recovery_sleep_hybrid"s,
+    "osd_recovery_sleep_degraded"s,
+    "osd_recovery_sleep_degraded_hdd"s,
+    "osd_recovery_sleep_degraded_ssd"s,
+    "osd_recovery_sleep_degraded_hybrid"s,
     "osd_delete_sleep"s,
     "osd_delete_sleep_hdd"s,
     "osd_delete_sleep_ssd"s,
@@ -10079,7 +10101,11 @@ void OSD::handle_conf_change(const ConfigProxy& conf,
       changed.count("osd_recovery_sleep") ||
       changed.count("osd_recovery_sleep_hdd") ||
       changed.count("osd_recovery_sleep_ssd") ||
-      changed.count("osd_recovery_sleep_hybrid")) {
+      changed.count("osd_recovery_sleep_hybrid") ||
+      changed.count("osd_recovery_sleep_degraded") ||
+      changed.count("osd_recovery_sleep_degraded_hdd") ||
+      changed.count("osd_recovery_sleep_degraded_ssd") ||
+      changed.count("osd_recovery_sleep_degraded_hybrid")) {
     maybe_override_sleep_options_for_qos();
   }
   if (changed.count("osd_min_recovery_priority")) {
@@ -10411,6 +10437,12 @@ void OSD::maybe_override_sleep_options_for_qos()
     cct->_conf.set_val("osd_recovery_sleep_ssd", std::to_string(0));
     cct->_conf.set_val("osd_recovery_sleep_hybrid", std::to_string(0));
 
+    // Disable recovery sleep for pg degraded
+    cct->_conf.set_val("osd_recovery_sleep_degraded", std::to_string(0));
+    cct->_conf.set_val("osd_recovery_sleep_degraded_hdd", std::to_string(0));
+    cct->_conf.set_val("osd_recovery_sleep_degraded_ssd", std::to_string(0));
+    cct->_conf.set_val("osd_recovery_sleep_degraded_hybrid", std::to_string(0));
+
     // Disable delete sleep
     cct->_conf.set_val("osd_delete_sleep", std::to_string(0));
     cct->_conf.set_val("osd_delete_sleep_hdd", std::to_string(0));

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 0a4928fc20af6a26de8901562dd773f3ff942d9b..1c8c4107b2767eac096bbd15562da5f1dadc8763 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -2017,6 +2017,7 @@ private:
   int get_num_op_threads();
 
   float get_osd_recovery_sleep();
+  float get_osd_recovery_sleep_degraded();
   float get_osd_delete_sleep();
   float get_osd_snap_trim_sleep();
 

diff --git a/src/osd/PG.h b/src/osd/PG.h
index bb8caa36b9544abb6390533e1ab9466ef2f4a461..d27d8196ad3467a942a5e1ddd29248e67da72a98 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1241,6 +1241,7 @@ protected:
 
 public:
   int pg_stat_adjust(osd_stat_t *new_stat);
+  bool is_degraded() const { return recovery_state.is_degraded(); }
 protected:
   bool delete_needs_sleep = false;
 
@@ -1264,7 +1265,6 @@ protected:
   bool is_backfill_unfound() const { return recovery_state.is_backfill_unfound(); }
   bool is_incomplete() const { return recovery_state.is_incomplete(); }
   bool is_clean() const { return recovery_state.is_clean(); }
-  bool is_degraded() const { return recovery_state.is_degraded(); }
   bool is_undersized() const { return recovery_state.is_undersized(); }
   bool is_scrubbing() const { return state_test(PG_STATE_SCRUBBING); } // Primary only
   bool is_remapped() const { return recovery_state.is_remapped(); }