mon: MonitorDBStore: allow randomly injecting random delays on writes

Adds two new config options:

mon_inject_transaction_delay_probability : DOUBLE (0.0-1.0, default: 0.0)
mon_inject_transaction_delay_max : DOUBLE (seconds, default: 10.0)

If probability is set to a value greater than 0, just before applying
the transaction, the store will decide whether to inject a delay,
randomly choosing a value between 0 and the max.

Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
(cherry picked from commit beaa04e4119765d5775a6c48fd072dd95c984e3b)

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 2c4f4da4fa38225eb1b7ce8a0e1efa82136cd46e..c6168d4a39fde4cebf10b7e46df1f57327576bde 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -221,6 +221,8 @@ OPTION(mon_mds_force_trim_to, OPT_INT, 0)   // force mon to trim mdsmaps to this
 // dump transactions
 OPTION(mon_debug_dump_transactions, OPT_BOOL, false)
 OPTION(mon_debug_dump_location, OPT_STR, "/var/log/ceph/$cluster-$name.tdump")
+OPTION(mon_inject_transaction_delay_max, OPT_DOUBLE, 10.0)      // seconds
+OPTION(mon_inject_transaction_delay_probability, OPT_DOUBLE, 0) // range [0, 1]
 
 OPTION(mon_sync_provider_kill_at, OPT_INT, 0)  // kill the sync provider at a specific point in the work flow
 OPTION(mon_sync_requester_kill_at, OPT_INT, 0) // kill the sync requester at a specific point in the work flow

diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index a0c82b7c8b6efed7abcfd8ba1868b7f622f499d5..e37b95db9008584e8113b45e81b70476dc8665f1 100644 (file)
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -304,6 +304,24 @@ class MonitorDBStore
       : store(s), t(t), oncommit(f)
     {}
     void finish(int r) {
+      /* The store serializes writes.  Each transaction is handled
+       * sequentially by the io_work Finisher.  If a transaction takes longer
+       * to apply its state to permanent storage, then no other transaction
+       * will be handled meanwhile.
+       *
+       * We will now randomly inject random delays.  We can safely sleep prior
+       * to applying the transaction as it won't break the model.
+       */
+      double delay_prob = g_conf->mon_inject_transaction_delay_probability;
+      if (delay_prob && (rand() % 10000 < delay_prob * 10000.0)) {
+        utime_t delay;
+        double delay_max = g_conf->mon_inject_transaction_delay_max;
+        delay.set_from_double(delay_max * (double)(rand() % 10000) / 10000.0);
+        lsubdout(g_ceph_context, mon, 1)
+          << "apply_transaction will be delayed for " << delay
+          << " seconds" << dendl;
+        delay.sleep();
+      }
       int ret = store->apply_transaction(t);
       oncommit->complete(ret);
     }