os/bluestore/BlockDevice: inject block failures

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index c686de938496867071acf0a7e21dee8c9bdc3ee3..1a609242ab4a16d1a281f4774e8efe5593bcc905 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -833,6 +833,7 @@ OPTION(memstore_page_set, OPT_BOOL, true)
 OPTION(memstore_page_size, OPT_U64, 64 << 10)
 
 OPTION(bdev_debug_inflight_ios, OPT_BOOL, false)
+OPTION(bdev_inject_crash, OPT_INT, 0)  // if N>0, then ~ 1/N IOs will complete before we crash on flush.
 
 OPTION(bluefs_alloc_size, OPT_U64, 1048576)
 OPTION(bluefs_max_prefetch, OPT_U64, 1048576)

diff --git a/src/os/bluestore/BlockDevice.cc b/src/os/bluestore/BlockDevice.cc
index aeed02adc805ed986052b27ed8da534cc0126d2b..618b6cfef25ac13dbe3401ebbf2d5d34c022108e 100644 (file)
--- a/src/os/bluestore/BlockDevice.cc
+++ b/src/os/bluestore/BlockDevice.cc
@@ -141,6 +141,14 @@ void BlockDevice::close()
 int BlockDevice::flush()
 {
   dout(10) << __func__ << " start" << dendl;
+  if (g_conf->bdev_inject_crash) {
+    // sleep for a moment to give other threads a chance to submit or
+    // wait on io that races with a flush.
+    derr << __func__ << " injecting crash. first we sleep..." << dendl;
+    sleep(3);
+    derr << __func__ << " and now we die" << dendl;
+    assert(0 == "bdev_inject_crash");
+  }
   utime_t start = ceph_clock_now(NULL);
   int r = ::fdatasync(fd);
   utime_t end = ceph_clock_now(NULL);
@@ -322,13 +330,22 @@ int BlockDevice::aio_write(
     ioc->pending_aios.push_back(FS::aio_t(ioc, fd));
     ioc->num_pending.inc();
     FS::aio_t& aio = ioc->pending_aios.back();
-    bl.prepare_iov(&aio.iov);
-    for (unsigned i=0; i<aio.iov.size(); ++i) {
-      dout(30) << "aio " << i << " " << aio.iov[i].iov_base
-              << " " << aio.iov[i].iov_len << dendl;
+    if (g_conf->bdev_inject_crash &&
+       rand() % g_conf->bdev_inject_crash == 0) {
+      derr << __func__ << " bdev_inject_crash: dropping io " << off << "~" << len
+          << dendl;
+      // generate a real io so that aio_wait behaves properly, but make it
+      // a read instead of write, and toss the result.
+      aio.pread(off, len);
+    } else {
+      bl.prepare_iov(&aio.iov);
+      for (unsigned i=0; i<aio.iov.size(); ++i) {
+       dout(30) << "aio " << i << " " << aio.iov[i].iov_base
+                << " " << aio.iov[i].iov_len << dendl;
+      }
+      aio.bl.claim_append(bl);
+      aio.pwritev(off);
     }
-    aio.bl.claim_append(bl);
-    aio.pwritev(off);
     dout(2) << __func__ << " prepared aio " << &aio << dendl;
   } else
 #endif

diff --git a/src/os/fs/FS.h b/src/os/fs/FS.h
index 0249c4cd052ce33852f3aea4badf1cdcd4b720eb..af8db21fefd478634a1adea87142ddecb55da50c 100644 (file)
--- a/src/os/fs/FS.h
+++ b/src/os/fs/FS.h
@@ -72,6 +72,13 @@ public:
       for (unsigned u=0; u<iov.size(); ++u)
        length += iov[u].iov_len;
     }
+    void pread(uint64_t _offset, uint64_t len) {
+      offset = _offset;
+      length = len;
+      bufferptr p(length);
+      bl.append(p);
+      io_prep_pread(&iocb, fd, p.c_str(), length, offset);
+    }
 
     int get_return_value() {
       return rval;