From: Joao Eduardo Luis <joao@redhat.com>
Date: Tue, 9 Dec 2014 17:35:47 +0000 (+0000)
Subject: mon: MonitorDBStore: allow randomly injecting random delays on writes
X-Git-Tag: v0.91~55^2~10^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=beaa04e4119765d5775a6c48fd072dd95c984e3b;p=ceph.git

mon: MonitorDBStore: allow randomly injecting random delays on writes

Adds two new config options:

mon_inject_transaction_delay_probability : DOUBLE (0.0-1.0, default: 0.0)
mon_inject_transaction_delay_max : DOUBLE (seconds, default: 10.0)

If probability is set to a value greater than 0, just before applying
the transaction, the store will decide whether to inject a delay,
randomly choosing a value between 0 and the max.

Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
---

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index d34a147b4c9f..c804e1a516ea 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -226,6 +226,8 @@ OPTION(mon_mds_force_trim_to, OPT_INT, 0)   // force mon to trim mdsmaps to this
 // dump transactions
 OPTION(mon_debug_dump_transactions, OPT_BOOL, false)
 OPTION(mon_debug_dump_location, OPT_STR, "/var/log/ceph/$cluster-$name.tdump")
+OPTION(mon_inject_transaction_delay_max, OPT_DOUBLE, 10.0)      // seconds
+OPTION(mon_inject_transaction_delay_probability, OPT_DOUBLE, 0) // range [0, 1]
 
 OPTION(mon_sync_provider_kill_at, OPT_INT, 0)  // kill the sync provider at a specific point in the work flow
 OPTION(mon_sync_requester_kill_at, OPT_INT, 0) // kill the sync requester at a specific point in the work flow
diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index a0c82b7c8b6e..e37b95db9008 100644
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -304,6 +304,24 @@ class MonitorDBStore
       : store(s), t(t), oncommit(f)
     {}
     void finish(int r) {
+      /* The store serializes writes.  Each transaction is handled
+       * sequentially by the io_work Finisher.  If a transaction takes longer
+       * to apply its state to permanent storage, then no other transaction
+       * will be handled meanwhile.
+       *
+       * We will now randomly inject random delays.  We can safely sleep prior
+       * to applying the transaction as it won't break the model.
+       */
+      double delay_prob = g_conf->mon_inject_transaction_delay_probability;
+      if (delay_prob && (rand() % 10000 < delay_prob * 10000.0)) {
+        utime_t delay;
+        double delay_max = g_conf->mon_inject_transaction_delay_max;
+        delay.set_from_double(delay_max * (double)(rand() % 10000) / 10000.0);
+        lsubdout(g_ceph_context, mon, 1)
+          << "apply_transaction will be delayed for " << delay
+          << " seconds" << dendl;
+        delay.sleep();
+      }
       int ret = store->apply_transaction(t);
       oncommit->complete(ret);
     }