From 5fb09a9ac9c9c1bec5e03c9d5a3a1c38f63b0fe2 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Thu, 3 Dec 2015 20:03:10 -0500
Subject: [PATCH] os/bluestore/BlockDevice: inject block failures

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/common/config_opts.h        |  1 +
 src/os/bluestore/BlockDevice.cc | 29 +++++++++++++++++++++++------
 src/os/fs/FS.h                  |  7 +++++++
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index c686de938496..1a609242ab4a 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -833,6 +833,7 @@ OPTION(memstore_page_set, OPT_BOOL, true)
 OPTION(memstore_page_size, OPT_U64, 64 << 10)
 
 OPTION(bdev_debug_inflight_ios, OPT_BOOL, false)
+OPTION(bdev_inject_crash, OPT_INT, 0)  // if N>0, then ~ 1/N IOs will complete before we crash on flush.
 
 OPTION(bluefs_alloc_size, OPT_U64, 1048576)
 OPTION(bluefs_max_prefetch, OPT_U64, 1048576)
diff --git a/src/os/bluestore/BlockDevice.cc b/src/os/bluestore/BlockDevice.cc
index aeed02adc805..618b6cfef25a 100644
--- a/src/os/bluestore/BlockDevice.cc
+++ b/src/os/bluestore/BlockDevice.cc
@@ -141,6 +141,14 @@ void BlockDevice::close()
 int BlockDevice::flush()
 {
   dout(10) << __func__ << " start" << dendl;
+  if (g_conf->bdev_inject_crash) {
+    // sleep for a moment to give other threads a chance to submit or
+    // wait on io that races with a flush.
+    derr << __func__ << " injecting crash. first we sleep..." << dendl;
+    sleep(3);
+    derr << __func__ << " and now we die" << dendl;
+    assert(0 == "bdev_inject_crash");
+  }
   utime_t start = ceph_clock_now(NULL);
   int r = ::fdatasync(fd);
   utime_t end = ceph_clock_now(NULL);
@@ -322,13 +330,22 @@ int BlockDevice::aio_write(
     ioc->pending_aios.push_back(FS::aio_t(ioc, fd));
     ioc->num_pending.inc();
     FS::aio_t& aio = ioc->pending_aios.back();
-    bl.prepare_iov(&aio.iov);
-    for (unsigned i=0; i<aio.iov.size(); ++i) {
-      dout(30) << "aio " << i << " " << aio.iov[i].iov_base
-	       << " " << aio.iov[i].iov_len << dendl;
+    if (g_conf->bdev_inject_crash &&
+	rand() % g_conf->bdev_inject_crash == 0) {
+      derr << __func__ << " bdev_inject_crash: dropping io " << off << "~" << len
+	   << dendl;
+      // generate a real io so that aio_wait behaves properly, but make it
+      // a read instead of write, and toss the result.
+      aio.pread(off, len);
+    } else {
+      bl.prepare_iov(&aio.iov);
+      for (unsigned i=0; i<aio.iov.size(); ++i) {
+	dout(30) << "aio " << i << " " << aio.iov[i].iov_base
+		 << " " << aio.iov[i].iov_len << dendl;
+      }
+      aio.bl.claim_append(bl);
+      aio.pwritev(off);
     }
-    aio.bl.claim_append(bl);
-    aio.pwritev(off);
     dout(2) << __func__ << " prepared aio " << &aio << dendl;
   } else
 #endif
diff --git a/src/os/fs/FS.h b/src/os/fs/FS.h
index 0249c4cd052c..af8db21fefd4 100644
--- a/src/os/fs/FS.h
+++ b/src/os/fs/FS.h
@@ -72,6 +72,13 @@ public:
       for (unsigned u=0; u<iov.size(); ++u)
 	length += iov[u].iov_len;
     }
+    void pread(uint64_t _offset, uint64_t len) {
+      offset = _offset;
+      length = len;
+      bufferptr p(length);
+      bl.append(p);
+      io_prep_pread(&iocb, fd, p.c_str(), length, offset);
+    }
 
     int get_return_value() {
       return rval;
-- 
2.47.3