os/FileStore: add stall injection into filestore op queue

Allow admin to artificially induce a stall in the op queue.  Forces the
thread(s) to sleep for N seconds.  We pause for 1 second increments and
recheck the value so that a previously stalled thread can be unwedged by
reinjecting a lower value (or 0).  To stall indefinitely, just injust
very large number.

Signed-off-by: Sage Weil <sage@inktank.com>
(cherry picked from commit 657df852e9c89bfacdbce25ea014f7830d61e6aa)

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index d72a2c49e73caa7da2cff6f0a9fc75904dcdbe15..627a050ffbb71a9045a0843ce95f58426f239358 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -425,6 +425,7 @@ OPTION(filestore_update_to, OPT_INT, 1000)
 OPTION(filestore_blackhole, OPT_BOOL, false)     // drop any new transactions on the floor
 OPTION(filestore_dump_file, OPT_STR, "")         // file onto which store transaction dumps
 OPTION(filestore_kill_at, OPT_INT, 0)            // inject a failure at the n'th opportunity
+OPTION(filestore_inject_stall, OPT_INT, 0)       // artificially stall for N seconds in op queue thread
 OPTION(filestore_fail_eio, OPT_BOOL, true)       // fail/crash on EIO
 OPTION(journal_dio, OPT_BOOL, true)
 OPTION(journal_aio, OPT_BOOL, false)

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 5976e68b6d297791994389bbc5f8461821bb0479..89d9cbf5445c64a6865472b925abf858b16ba4e5 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -1948,6 +1948,16 @@ void FileStore::op_queue_release_throttle(Op *o)
 
 void FileStore::_do_op(OpSequencer *osr)
 {
+  // inject a stall?
+  if (g_conf->filestore_inject_stall) {
+    int orig = g_conf->filestore_inject_stall;
+    dout(5) << "_do_op filestore_inject_stall " << orig << ", sleeping" << dendl;
+    for (int n = 0; n < g_conf->filestore_inject_stall; n++)
+      sleep(1);
+    g_conf->set_val("filestore_inject_stall", "0");
+    dout(5) << "_do_op done stalling" << dendl;
+  }
+
   osr->apply_lock.Lock();
   Op *o = osr->peek_queue();
   apply_manager.op_apply_start(o->op);