From: Sage Weil <sage@redhat.com>
Date: Fri, 20 Sep 2019 13:10:59 +0000 (-0500)
Subject: os/bluestore: fix objectstore_blackhole read-after-write
X-Git-Tag: v14.2.5~67^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d004126bd1ec8b492e2c0eeb644a6594c0066de4;p=ceph.git

os/bluestore: fix objectstore_blackhole read-after-write

Instead of tossing out transactions from queue_transactions (which prevents
us from reading back the results), instead (1) stop doing any write IO, and
(2) stop trimming the cache.

Fixes: https://tracker.ceph.com/issues/40684
Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 6c2a8e472dc71b962d7de008e30631f125b148c3)

 Conflicts:
	src/os/bluestore/BlueStore.h
   Due to cache refactoring one should handle objectstore_blackhole param
   inside Cache:trim/trim_all funcs.
---

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 60baa85da099..e8e5e517f894 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -873,12 +873,18 @@ BlueStore::Cache *BlueStore::Cache::create(CephContext* cct, string type,
 void BlueStore::Cache::trim(uint64_t onode_max, uint64_t buffer_max)
 {
   std::lock_guard l(lock);
+  if (cct->_conf->objectstore_blackhole) {
+    // do not trim if we are throwing away IOs a layer down
+    return;
+  }
   _trim(onode_max, buffer_max);
 }
 
 void BlueStore::Cache::trim_all()
 {
   std::lock_guard l(lock);
+  // we should not be shutting down after the blackhole is enabled
+  assert(!cct->_conf->objectstore_blackhole);
   _trim(0, 0);
 }
 
@@ -10991,16 +10997,6 @@ int BlueStore::queue_transactions(
   ObjectStore::Transaction::collect_contexts(
     tls, &on_applied, &on_commit, &on_applied_sync);
 
-  if (cct->_conf->objectstore_blackhole) {
-    dout(0) << __func__ << " objectstore_blackhole = TRUE, dropping transaction"
-	    << dendl;
-    for (auto& l : { on_applied, on_commit, on_applied_sync }) {
-      for (auto c : l) {
-	delete c;
-      }
-    }
-    return 0;
-  }
   auto start = mono_clock::now();
 
   Collection *c = static_cast<Collection*>(ch.get());
diff --git a/src/os/bluestore/KernelDevice.cc b/src/os/bluestore/KernelDevice.cc
index 3530c454bf27..b12030498839 100644
--- a/src/os/bluestore/KernelDevice.cc
+++ b/src/os/bluestore/KernelDevice.cc
@@ -808,6 +808,11 @@ int KernelDevice::write(
 	   << (buffered ? " (buffered)" : " (direct)")
 	   << dendl;
   ceph_assert(is_valid_io(off, len));
+  if (cct->_conf->objectstore_blackhole) {
+    lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO"
+	       << dendl;
+    return 0;
+  }
 
   if ((!buffered || bl.get_num_buffers() >= IOV_MAX) &&
       bl.rebuild_aligned_size_and_memory(block_size, block_size, IOV_MAX)) {
@@ -832,6 +837,11 @@ int KernelDevice::aio_write(
 	   << (buffered ? " (buffered)" : " (direct)")
 	   << dendl;
   ceph_assert(is_valid_io(off, len));
+  if (cct->_conf->objectstore_blackhole) {
+    lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO"
+	       << dendl;
+    return 0;
+  }
 
   if ((!buffered || bl.get_num_buffers() >= IOV_MAX) &&
       bl.rebuild_aligned_size_and_memory(block_size, block_size, IOV_MAX)) {
@@ -908,6 +918,11 @@ int KernelDevice::aio_write(
 int KernelDevice::discard(uint64_t offset, uint64_t len)
 {
   int r = 0;
+  if (cct->_conf->objectstore_blackhole) {
+    lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO"
+	       << dendl;
+    return 0;
+  }
   if (support_discard) {
       dout(10) << __func__
 	       << " 0x" << std::hex << offset << "~" << len << std::dec