osd: Add support for osd_delete_sleep to throttle pg deletion

Fixes: http://tracker.ceph.com/issues/36474
Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 92f1f5859f142cb83cc1d1a367853ac91d6b01d4)

Conflicts:
    src/osd/OSD.cc
- use cct->_conf->get_val instead of cct->_conf.get_val

diff --git a/src/common/options.cc b/src/common/options.cc
index b2246579eb8c649b0d0314814d90c4241932218e..5cb70d742e45040ed5a53d3407b63b0d38282966 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3323,6 +3323,22 @@ std::vector<Option> get_global_options() {
     .set_default(30)
     .set_description(""),
 
+    Option("osd_delete_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("Time in seconds to sleep before next removal transaction (overrides values below)"),
+
+    Option("osd_delete_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(5)
+    .set_description("Time in seconds to sleep before next removal transaction for HDDs"),
+
+    Option("osd_delete_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("Time in seconds to sleep before next removal transaction for SSDs"),
+
+    Option("osd_delete_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(2)
+    .set_description("Time in seconds to sleep before next removal transaction when data is on HDD and journal is on SSD"),
+
     Option("osd_failsafe_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(.97)
     .set_description(""),

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 1fb8ca036c0f6653c4dbce5d9b7f3fd522a4f7c3..71bf48820edf6b953b42099f46a3b52c65092aab 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2388,6 +2388,18 @@ float OSD::get_osd_recovery_sleep()
     return cct->_conf->osd_recovery_sleep_hdd;
 }
 
+float OSD::get_osd_delete_sleep()
+{
+  float osd_delete_sleep = cct->_conf->get_val<double>("osd_delete_sleep");
+  if (osd_delete_sleep > 0)
+    return osd_delete_sleep;
+  if (!store_is_rotational && !journal_is_rotational)
+    return cct->_conf->get_val<double>("osd_delete_sleep_ssd");
+  if (store_is_rotational && !journal_is_rotational)
+    return cct->_conf->get_val<double>("osd_delete_sleep_hybrid");
+  return cct->_conf->get_val<double>("osd_delete_sleep_hdd");
+}
+
 int OSD::init()
 {
   CompatSet initial, diff;

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index c142b63747a0c60125319d6e4eda0771404f71af..8b4aa0dba00f727617bdb559d6438c02b707069f 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -2217,6 +2217,7 @@ private:
   int get_num_op_threads();
 
   float get_osd_recovery_sleep();
+  float get_osd_delete_sleep();
 
   void probe_smart(ostream& ss);
   int probe_smart_device(const char *device, int timeout, std::string *result);

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index c333715878821395aec0b1c6babc6c0932808a18..1c765d3336761eb71605e47bcf64eaa5a3d74247 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -6531,6 +6531,35 @@ void PG::_delete_some(ObjectStore::Transaction *t)
 {
   dout(10) << __func__ << dendl;
 
+  {
+    float osd_delete_sleep = osd->osd->get_osd_delete_sleep();
+    if (osd_delete_sleep > 0 && delete_needs_sleep) {
+      epoch_t e = get_osdmap()->get_epoch();
+      PGRef pgref(this);
+      auto delete_requeue_callback = new FunctionContext([this, pgref, e](int r) {
+        dout(20) << __func__ << " wake up at "
+                 << ceph_clock_now()
+                << ", re-queuing delete" << dendl;
+        lock();
+        delete_needs_sleep = false;
+        if (!pg_has_reset_since(e)) {
+          osd->queue_for_pg_delete(get_pgid(), e);
+        }
+        unlock();
+      });
+
+      utime_t delete_schedule_time = ceph_clock_now();
+      delete_schedule_time += osd_delete_sleep;
+      Mutex::Locker l(osd->sleep_lock);
+      osd->sleep_timer.add_event_at(delete_schedule_time,
+                                               delete_requeue_callback);
+      dout(20) << __func__ << " Delete scheduled at " << delete_schedule_time << dendl;
+      return;
+    }
+  }
+
+  delete_needs_sleep = true;
+
   vector<ghobject_t> olist;
   int max = std::min(osd->store->get_ideal_list_max(),
                     (int)cct->_conf->osd_target_transaction_size);

diff --git a/src/osd/PG.h b/src/osd/PG.h
index bcd884a435734576d8aaa471346fcd402b76526d..ffb50360ee28a5d781305d78f7de8ae1389a8213 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -2743,6 +2743,7 @@ protected:
 
   /// most recently consumed osdmap's require_osd_version
   unsigned last_require_osd_release = 0;
+  bool delete_needs_sleep = false;
 
 protected:
   void reset_min_peer_features() {