osd: randomize deep scrubbing

osd_scrub_interval_randomize_ratio works to randomize the shallow scrubs
but doesn't prevent a thundering herd of deep scrubs every osd_deep_scrub_interval.
Add the option osd_deep_scrub_randomize_ratio which defines the rate at which scrubs
will randomly turn into deep scrubs.

Backports: hammer, infernalis

Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
Signed-off-by: Herve Rousseau <herve.rousseau@cern.ch>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index dbad270a3b9bfeed8f3f4c190e0ddf50f9afee6b..5ce5c76400b67c7e935bee7cc056af15e9e4821f 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -684,6 +684,7 @@ OPTION(osd_scrub_sleep, OPT_FLOAT, 0)   // sleep between [deep]scrub ops
 OPTION(osd_scrub_auto_repair, OPT_BOOL, false)   // whether auto-repair inconsistencies upon deep-scrubbing
 OPTION(osd_scrub_auto_repair_num_errors, OPT_U32, 5)   // only auto-repair when number of errors is below this threshold
 OPTION(osd_deep_scrub_interval, OPT_FLOAT, 60*60*24*7) // once a week
+OPTION(osd_deep_scrub_randomize_ratio, OPT_FLOAT, 0.15) // scrubs will randomly become deep scrubs at this rate (0.15 -> 15% of scrubs are deep)
 OPTION(osd_deep_scrub_stride, OPT_INT, 524288)
 OPTION(osd_deep_scrub_update_digest_min_age, OPT_INT, 2*60*60)   // objects must be this old (seconds) before we update the whole-object digest on scrub
 OPTION(osd_scan_list_ping_tp_interval, OPT_U64, 100)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 643f50fd20f199863d900f2f54d01a301aa48f08..571c431739a46e6ced1f76e2d59d059eb8e59bff 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -3246,6 +3246,11 @@ bool PG::sched_scrub()
   bool time_for_deep = (ceph_clock_now(cct) >=
     info.history.last_deep_scrub_stamp + cct->_conf->osd_deep_scrub_interval);
 
+  bool deep_coin_flip = (rand() % 100) < cct->_conf->osd_deep_scrub_randomize_ratio * 100;
+  dout(20) << __func__ << ": time_for_deep=" << time_for_deep << " deep_coin_flip=" << deep_coin_flip << dendl;
+
+  time_for_deep = (time_for_deep || deep_coin_flip);
+
   //NODEEP_SCRUB so ignore time initiated deep-scrub
   if (osd->osd->get_osdmap()->test_flag(CEPH_OSDMAP_NODEEP_SCRUB) ||
       pool.info.has_flag(pg_pool_t::FLAG_NODEEP_SCRUB))