osd: inject simple sleep in recovery

This is clearly not the most sophisticated way to throttle recovery, but
in the grand tradition of osd_scrub_sleep and osd_snap_trim_sleep it is a
crude tool that can be quite effective in the meantime.

Backport: hammer, firefly
Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index f914ca93fd0acdf9eeb1afa2b9219dcc8d3cc45b..51ff92d598e65a09f4763a16b4ba01d9e9e738d9 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -558,6 +558,7 @@ OPTION(osd_backfill_scan_min, OPT_INT, 64)
 OPTION(osd_backfill_scan_max, OPT_INT, 512)
 OPTION(osd_op_thread_timeout, OPT_INT, 15)
 OPTION(osd_recovery_thread_timeout, OPT_INT, 30)
+OPTION(osd_recovery_sleep, OPT_FLOAT, 0)         // seconds to sleep between recovery ops
 OPTION(osd_snap_trim_thread_timeout, OPT_INT, 60*60*1)
 OPTION(osd_snap_trim_sleep, OPT_FLOAT, 0)
 OPTION(osd_scrub_thread_timeout, OPT_INT, 60)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 9ce5a44cbeb0bc1718ec0ba6d52deb677eda9dcf..136c89de36fb9574f0b8a7e7fade9c0ca62fa904 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7820,6 +7820,13 @@ bool OSD::_recover_now()
 
 void OSD::do_recovery(PG *pg, ThreadPool::TPHandle &handle)
 {
+  if (g_conf->osd_recovery_sleep > 0) {
+    utime_t t;
+    t.set_from_double(g_conf->osd_recovery_sleep);
+    t.sleep();
+    dout(20) << __func__ << " slept for " << t << dendl;
+  }
+
   // see how many we should try to start.  note that this is a bit racy.
   recovery_wq.lock();
   int max = MIN(cct->_conf->osd_recovery_max_active - recovery_ops_active,