From d004126bd1ec8b492e2c0eeb644a6594c0066de4 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 20 Sep 2019 08:10:59 -0500
Subject: [PATCH] os/bluestore: fix objectstore_blackhole read-after-write

Instead of tossing out transactions from queue_transactions (which prevents
us from reading back the results), instead (1) stop doing any write IO, and
(2) stop trimming the cache.

Fixes: https://tracker.ceph.com/issues/40684
Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 6c2a8e472dc71b962d7de008e30631f125b148c3)

 Conflicts:
	src/os/bluestore/BlueStore.h
   Due to cache refactoring one should handle objectstore_blackhole param
   inside Cache:trim/trim_all funcs.
---
 src/os/bluestore/BlueStore.cc    | 16 ++++++----------
 src/os/bluestore/KernelDevice.cc | 15 +++++++++++++++
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 60baa85da09..e8e5e517f89 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -873,12 +873,18 @@ BlueStore::Cache *BlueStore::Cache::create(CephContext* cct, string type,
 void BlueStore::Cache::trim(uint64_t onode_max, uint64_t buffer_max)
 {
   std::lock_guard l(lock);
+  if (cct->_conf->objectstore_blackhole) {
+    // do not trim if we are throwing away IOs a layer down
+    return;
+  }
   _trim(onode_max, buffer_max);
 }
 
 void BlueStore::Cache::trim_all()
 {
   std::lock_guard l(lock);
+  // we should not be shutting down after the blackhole is enabled
+  assert(!cct->_conf->objectstore_blackhole);
   _trim(0, 0);
 }
 
@@ -10991,16 +10997,6 @@ int BlueStore::queue_transactions(
   ObjectStore::Transaction::collect_contexts(
     tls, &on_applied, &on_commit, &on_applied_sync);
 
-  if (cct->_conf->objectstore_blackhole) {
-    dout(0) << __func__ << " objectstore_blackhole = TRUE, dropping transaction"
-	    << dendl;
-    for (auto& l : { on_applied, on_commit, on_applied_sync }) {
-      for (auto c : l) {
-	delete c;
-      }
-    }
-    return 0;
-  }
   auto start = mono_clock::now();
 
   Collection *c = static_cast<Collection*>(ch.get());
diff --git a/src/os/bluestore/KernelDevice.cc b/src/os/bluestore/KernelDevice.cc
index 3530c454bf2..b1203049883 100644
--- a/src/os/bluestore/KernelDevice.cc
+++ b/src/os/bluestore/KernelDevice.cc
@@ -808,6 +808,11 @@ int KernelDevice::write(
 	   << (buffered ? " (buffered)" : " (direct)")
 	   << dendl;
   ceph_assert(is_valid_io(off, len));
+  if (cct->_conf->objectstore_blackhole) {
+    lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO"
+	       << dendl;
+    return 0;
+  }
 
   if ((!buffered || bl.get_num_buffers() >= IOV_MAX) &&
       bl.rebuild_aligned_size_and_memory(block_size, block_size, IOV_MAX)) {
@@ -832,6 +837,11 @@ int KernelDevice::aio_write(
 	   << (buffered ? " (buffered)" : " (direct)")
 	   << dendl;
   ceph_assert(is_valid_io(off, len));
+  if (cct->_conf->objectstore_blackhole) {
+    lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO"
+	       << dendl;
+    return 0;
+  }
 
   if ((!buffered || bl.get_num_buffers() >= IOV_MAX) &&
       bl.rebuild_aligned_size_and_memory(block_size, block_size, IOV_MAX)) {
@@ -908,6 +918,11 @@ int KernelDevice::aio_write(
 int KernelDevice::discard(uint64_t offset, uint64_t len)
 {
   int r = 0;
+  if (cct->_conf->objectstore_blackhole) {
+    lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO"
+	       << dendl;
+    return 0;
+  }
   if (support_discard) {
       dout(10) << __func__
 	       << " 0x" << std::hex << offset << "~" << len << std::dec
-- 
2.47.3