osd debug: Add testing support for random read EIO errors

filestore_debug_random_read_err   Specify % of EIO all filestore reads

Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 443ef8c1a8719afd17a6b6ad3e4c76b5d14a4dc0..9d3a418fa3307bf9edbf69a8fa1d884c8923aff0 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1178,6 +1178,7 @@ OPTION(bluestore_debug_fsck_abort, OPT_BOOL, false)
 OPTION(bluestore_debug_omit_kv_commit, OPT_BOOL, false)
 OPTION(bluestore_debug_permit_any_bdev_label, OPT_BOOL, false)
 OPTION(bluestore_shard_finishers, OPT_BOOL, false)
+OPTION(bluestore_debug_random_read_err, OPT_DOUBLE, 0)
 
 OPTION(kstore_max_ops, OPT_U64, 512)
 OPTION(kstore_max_bytes, OPT_U64, 64*1024*1024)
@@ -1221,6 +1222,7 @@ OPTION(filestore_index_retry_probability, OPT_DOUBLE, 0)
 
 // Allow object read error injection
 OPTION(filestore_debug_inject_read_err, OPT_BOOL, false)
+OPTION(filestore_debug_random_read_err, OPT_DOUBLE, 0)
 
 OPTION(filestore_debug_omap_check, OPT_BOOL, false) // Expensive debugging check on sync
 OPTION(filestore_omap_header_cache_size, OPT_INT, 1024)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 8861636327c9d6e6b88882759142542b50c9108c..60998e8482d553dd722aee63d6b7e7e6e86a1162 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -6201,6 +6201,10 @@ int BlueStore::read(
   if (r == 0 && _debug_data_eio(oid)) {
     r = -EIO;
     derr << __func__ << " " << c->cid << " " << oid << " INJECT EIO" << dendl;
+  } else if (cct->_conf->bluestore_debug_random_read_err &&
+    (rand() % (int)(cct->_conf->bluestore_debug_random_read_err * 100.0)) == 0) {
+    dout(0) << __func__ << ": inject random EIO" << dendl;
+    r = -EIO;
   }
   dout(10) << __func__ << " " << cid << " " << oid
           << " 0x" << std::hex << offset << "~" << length << std::dec

diff --git a/src/os/filestore/FileStore.cc b/src/os/filestore/FileStore.cc
index d2f3053195f24d2b0396141ccbad9a760c3b7b59..7120428bc0c8a935e03babff657238f942ed952e 100644 (file)
--- a/src/os/filestore/FileStore.cc
+++ b/src/os/filestore/FileStore.cc
@@ -3249,6 +3249,10 @@ int FileStore::read(
   if (cct->_conf->filestore_debug_inject_read_err &&
       debug_data_eio(oid)) {
     return -EIO;
+  } else if (cct->_conf->filestore_debug_random_read_err &&
+    (rand() % (int)(cct->_conf->filestore_debug_random_read_err * 100.0)) == 0) {
+    dout(0) << __func__ << ": inject random EIO" << dendl;
+    return -EIO;
   } else {
     tracepoint(objectstore, read_exit, got);
     return got;