From cbe8b5bca40fd63a382b1a903087e7c34b314985 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 22 Jan 2013 18:08:22 -0800 Subject: [PATCH] os/FileStore: add stall injection into filestore op queue Allow admin to artificially induce a stall in the op queue. Forces the thread(s) to sleep for N seconds. We pause for 1 second increments and recheck the value so that a previously stalled thread can be unwedged by reinjecting a lower value (or 0). To stall indefinitely, just injust very large number. Signed-off-by: Sage Weil (cherry picked from commit 657df852e9c89bfacdbce25ea014f7830d61e6aa) --- src/common/config_opts.h | 1 + src/os/FileStore.cc | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index d72a2c49e73ca..627a050ffbb71 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -425,6 +425,7 @@ OPTION(filestore_update_to, OPT_INT, 1000) OPTION(filestore_blackhole, OPT_BOOL, false) // drop any new transactions on the floor OPTION(filestore_dump_file, OPT_STR, "") // file onto which store transaction dumps OPTION(filestore_kill_at, OPT_INT, 0) // inject a failure at the n'th opportunity +OPTION(filestore_inject_stall, OPT_INT, 0) // artificially stall for N seconds in op queue thread OPTION(filestore_fail_eio, OPT_BOOL, true) // fail/crash on EIO OPTION(journal_dio, OPT_BOOL, true) OPTION(journal_aio, OPT_BOOL, false) diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index 5976e68b6d297..89d9cbf5445c6 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -1948,6 +1948,16 @@ void FileStore::op_queue_release_throttle(Op *o) void FileStore::_do_op(OpSequencer *osr) { + // inject a stall? + if (g_conf->filestore_inject_stall) { + int orig = g_conf->filestore_inject_stall; + dout(5) << "_do_op filestore_inject_stall " << orig << ", sleeping" << dendl; + for (int n = 0; n < g_conf->filestore_inject_stall; n++) + sleep(1); + g_conf->set_val("filestore_inject_stall", "0"); + dout(5) << "_do_op done stalling" << dendl; + } + osr->apply_lock.Lock(); Op *o = osr->peek_queue(); apply_manager.op_apply_start(o->op); -- 2.39.5