Fix not getting expected injected read error (#12793)

Summary:
**Context/Summary:**

https://github.com/facebook/rocksdb/pull/12713 accidentally removed the mechanism of ignoring injected read error on non-critical read path such as read from filter. IO failure in read from filter should not fail the read as we can always read from the actual file. Therefore error injection in filter read path does not need to lead to failure in Get() and crash test should allow that. Otherwise, we will get crash test error "Didn't get expected error from..."

Pull Request resolved: https://github.com/facebook/rocksdb/pull/12793

Test Plan: CI

Reviewed By: cbi42

Differential Revision: D58895393

Pulled By: hx235

fbshipit-source-id: 5b605d8446e0b8d4149cdbe6f4be3c7534d4acfa

diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc
index 90ddb6f26799ccdd185cbb5b921ed268c756e9de..6bb454abbc4053ba4b785db5bb573fac0ff22d20 100644 (file)
--- a/db_stress_tool/no_batched_ops_stress.cc
+++ b/db_stress_tool/no_batched_ops_stress.cc
@@ -528,6 +528,7 @@ class NonBatchedOpsStressTest : public StressTest {
           FaultInjectionIOType::kRead);
       fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kMetadataRead);
+      SharedState::ignore_read_error = false;
     }
 
     const ExpectedValue pre_read_expected_value =
@@ -537,7 +538,7 @@ class NonBatchedOpsStressTest : public StressTest {
         thread->shared->Get(rand_column_families[0], rand_keys[0]);
 
     int injected_error_count = 0;
-    if (fault_fs_guard) {
+    if (fault_fs_guard && !SharedState::ignore_read_error) {
       injected_error_count = GetMinInjectedErrorCount(
           fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kRead),
@@ -687,10 +688,11 @@ class NonBatchedOpsStressTest : public StressTest {
             FaultInjectionIOType::kRead);
         fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kMetadataRead);
+        SharedState::ignore_read_error = false;
       }
       db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(),
                     statuses.data());
-      if (fault_fs_guard) {
+      if (fault_fs_guard && !SharedState::ignore_read_error) {
         injected_error_count = GetMinInjectedErrorCount(
             fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
                 FaultInjectionIOType::kRead),
@@ -981,6 +983,7 @@ class NonBatchedOpsStressTest : public StressTest {
           FaultInjectionIOType::kRead);
       fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kMetadataRead);
+      SharedState::ignore_read_error = false;
     }
 
     Status s;
@@ -998,7 +1001,7 @@ class NonBatchedOpsStressTest : public StressTest {
         thread->shared->Get(column_family, key);
 
     int injected_error_count = 0;
-    if (fault_fs_guard) {
+    if (fault_fs_guard && !SharedState::ignore_read_error) {
       injected_error_count = GetMinInjectedErrorCount(
           fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kRead),
@@ -1384,6 +1387,7 @@ class NonBatchedOpsStressTest : public StressTest {
             FaultInjectionIOType::kRead);
         fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kMetadataRead);
+        SharedState::ignore_read_error = false;
       }
 
       std::vector<PinnableAttributeGroups> results;
@@ -1397,7 +1401,7 @@ class NonBatchedOpsStressTest : public StressTest {
       db_->MultiGetEntity(read_opts_copy, num_keys, key_slices.data(),
                           results.data());
 
-      if (fault_fs_guard) {
+      if (fault_fs_guard && !SharedState::ignore_read_error) {
         verify_expected_errors(
             [&](size_t i) { return results[i][0].status(); });
       }
@@ -1418,6 +1422,7 @@ class NonBatchedOpsStressTest : public StressTest {
             FaultInjectionIOType::kRead);
         fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kMetadataRead);
+        SharedState::ignore_read_error = false;
       }
 
       std::vector<PinnableWideColumns> results(num_keys);
@@ -1426,7 +1431,7 @@ class NonBatchedOpsStressTest : public StressTest {
       db_->MultiGetEntity(read_opts_copy, cfh, num_keys, key_slices.data(),
                           results.data(), statuses.data());
 
-      if (fault_fs_guard) {
+      if (fault_fs_guard && !SharedState::ignore_read_error) {
         verify_expected_errors([&](size_t i) { return statuses[i]; });
       }
 
@@ -1483,6 +1488,7 @@ class NonBatchedOpsStressTest : public StressTest {
           FaultInjectionIOType::kRead);
       fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kMetadataRead);
+      SharedState::ignore_read_error = false;
     }
 
     for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix);
@@ -1516,7 +1522,7 @@ class NonBatchedOpsStressTest : public StressTest {
     }
 
     int injected_error_count = 0;
-    if (fault_fs_guard) {
+    if (fault_fs_guard && !SharedState::ignore_read_error) {
       injected_error_count = GetMinInjectedErrorCount(
           fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kRead),