osd: add 'osd debug reject backfill probability' option

This will make the OSD randomly reject backfill reservation requests.  This
exercises the failure code paths but does not break overall behavior
because the primary will back off and retry later.

This should help us reproduce #7922.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index c364f3b77527e78616e5e80c4a440ee740899f39..6b97c82cfba945a97327eb67b5369c28665f5c0e 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -534,6 +534,7 @@ OPTION(osd_debug_op_order, OPT_BOOL, false)
 OPTION(osd_debug_verify_snaps_on_info, OPT_BOOL, false)
 OPTION(osd_debug_verify_stray_on_activate, OPT_BOOL, false)
 OPTION(osd_debug_skip_full_check_in_backfill_reservation, OPT_BOOL, false)
+OPTION(osd_debug_reject_backfill_probability, OPT_DOUBLE, 0)
 OPTION(osd_enable_op_tracker, OPT_BOOL, true) // enable/disable OSD op tracking
 OPTION(osd_op_history_size, OPT_U32, 20)    // Max number of completed ops to track
 OPTION(osd_op_history_duration, OPT_U32, 600) // Oldest completed op to track

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 39e4abf68bf83e972253fd084b47229f4f152f0d..579cb099e724499938e62996ecf54eb8c477039f 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -5827,9 +5827,13 @@ boost::statechart::result
 PG::RecoveryState::RepNotRecovering::react(const RequestBackfillPrio &evt)
 {
   PG *pg = context< RecoveryMachine >().pg;
-
   double ratio, max_ratio;
-  if (pg->osd->too_full_for_backfill(&ratio, &max_ratio) &&
+
+  if (g_conf->osd_debug_reject_backfill_probability > 0 &&
+      (rand()%1000 < (g_conf->osd_debug_reject_backfill_probability*1000.0))) {
+    dout(10) << "backfill reservation rejected: failure injection" << dendl;
+    post_event(RemoteReservationRejected());
+  } else if (pg->osd->too_full_for_backfill(&ratio, &max_ratio) &&
       !pg->cct->_conf->osd_debug_skip_full_check_in_backfill_reservation) {
     dout(10) << "backfill reservation rejected: full ratio is "
             << ratio << ", which is greater than max allowed ratio "