f.find(".restore") != std::string::npos) {
return target()->DeleteFile(f);
}
- return Status::OK();
+ // Rename the file instead of deletion to keep the history, and
+ // at the same time it is not visible to RocksDB.
+ return target()->RenameFile(f, f + "_renamed_");
}
// If true, all manifest files will not be delted in DeleteFile().
"Number of bytes for a user-defined timestamp. Currently, only "
"8-byte is supported");
+DEFINE_int32(open_metadata_write_fault_one_in, 0,
+ "On non-zero, enables fault injection on file metadata write "
+ "during DB reopen.");
+
#endif // GFLAGS
DECLARE_int32(continuous_verification_interval);
DECLARE_int32(read_fault_one_in);
DECLARE_int32(write_fault_one_in);
+DECLARE_int32(open_metadata_write_fault_one_in);
namespace ROCKSDB_NAMESPACE {
class StressTest;
static_cast<int>(FLAGS_level_compaction_dynamic_level_bytes));
fprintf(stdout, "Read fault one in : %d\n", FLAGS_read_fault_one_in);
fprintf(stdout, "Write fault one in : %d\n", FLAGS_write_fault_one_in);
+ fprintf(stdout, "Open metadata write fault one in:\n");
+ fprintf(stdout, " %d\n",
+ FLAGS_open_metadata_write_fault_one_in);
fprintf(stdout, "Sync fault injection : %d\n", FLAGS_sync_fault_injection);
fprintf(stdout, "Best efforts recovery : %d\n",
static_cast<int>(FLAGS_best_efforts_recovery));
new DbStressListener(FLAGS_db, options_.db_paths, cf_descriptors));
options_.create_missing_column_families = true;
if (!FLAGS_use_txn) {
+#ifndef NDEBUG
+ // Determine whether we need to ingest file metadata write failures
+ // during DB reopen. If it does, enable it.
+ // Only ingest metadata error if it is reopening, as initial open
+ // failure doesn't need to be handled.
+ // TODO cover transaction DB is not covered in this fault test too.
+ bool ingest_meta_error =
+ FLAGS_open_metadata_write_fault_one_in &&
+ fault_fs_guard
+ ->FileExists(FLAGS_db + "/CURRENT", IOOptions(), nullptr)
+ .ok();
+ if (ingest_meta_error) {
+ fault_fs_guard->EnableMetadataWriteErrorInjection();
+ fault_fs_guard->SetRandomMetadataWriteError(
+ FLAGS_open_metadata_write_fault_one_in);
+ }
+ while (true) {
+#endif // NDEBUG
#ifndef ROCKSDB_LITE
- // StackableDB-based BlobDB
- if (FLAGS_use_blob_db) {
- blob_db::BlobDBOptions blob_db_options;
- blob_db_options.min_blob_size = FLAGS_blob_db_min_blob_size;
- blob_db_options.bytes_per_sync = FLAGS_blob_db_bytes_per_sync;
- blob_db_options.blob_file_size = FLAGS_blob_db_file_size;
- blob_db_options.enable_garbage_collection = FLAGS_blob_db_enable_gc;
- blob_db_options.garbage_collection_cutoff = FLAGS_blob_db_gc_cutoff;
-
- blob_db::BlobDB* blob_db = nullptr;
- s = blob_db::BlobDB::Open(options_, blob_db_options, FLAGS_db,
- cf_descriptors, &column_families_, &blob_db);
- if (s.ok()) {
- db_ = blob_db;
- }
- } else
+ // StackableDB-based BlobDB
+ if (FLAGS_use_blob_db) {
+ blob_db::BlobDBOptions blob_db_options;
+ blob_db_options.min_blob_size = FLAGS_blob_db_min_blob_size;
+ blob_db_options.bytes_per_sync = FLAGS_blob_db_bytes_per_sync;
+ blob_db_options.blob_file_size = FLAGS_blob_db_file_size;
+ blob_db_options.enable_garbage_collection = FLAGS_blob_db_enable_gc;
+ blob_db_options.garbage_collection_cutoff = FLAGS_blob_db_gc_cutoff;
+
+ blob_db::BlobDB* blob_db = nullptr;
+ s = blob_db::BlobDB::Open(options_, blob_db_options, FLAGS_db,
+ cf_descriptors, &column_families_,
+ &blob_db);
+ if (s.ok()) {
+ db_ = blob_db;
+ }
+ } else
#endif // !ROCKSDB_LITE
- {
- if (db_preload_finished_.load() && FLAGS_read_only) {
- s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db, cf_descriptors,
- &column_families_, &db_);
- } else {
- s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors,
- &column_families_, &db_);
+ {
+ if (db_preload_finished_.load() && FLAGS_read_only) {
+ s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db,
+ cf_descriptors, &column_families_, &db_);
+ } else {
+ s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors,
+ &column_families_, &db_);
+ }
+ }
+
+#ifndef NDEBUG
+ if (ingest_meta_error) {
+ fault_fs_guard->DisableMetadataWriteErrorInjection();
+ if (!s.ok()) {
+ // After failure to opening a DB due to IO error, retry should
+ // successfully open the DB with correct data if no IO error shows
+ // up.
+ ingest_meta_error = false;
+
+ Random rand(static_cast<uint32_t>(FLAGS_seed));
+ if (rand.OneIn(2)) {
+ fault_fs_guard->DeleteFilesCreatedAfterLastDirSync(IOOptions(),
+ nullptr);
+ }
+ if (rand.OneIn(3)) {
+ fault_fs_guard->DropUnsyncedFileData();
+ } else if (rand.OneIn(2)) {
+ fault_fs_guard->DropRandomUnsyncedFileData(&rand);
+ }
+ continue;
+ }
}
+ break;
}
+#endif // NDEBUG
} else {
#ifndef ROCKSDB_LITE
TransactionDBOptions txn_db_options;
#ifndef NDEBUG
if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection ||
- FLAGS_write_fault_one_in) {
+ FLAGS_write_fault_one_in || FLAGS_open_metadata_write_fault_one_in) {
FaultInjectionTestFS* fs =
new FaultInjectionTestFS(raw_env->GetFileSystem());
fault_fs_guard.reset(fs);
"max_key_len": 3,
"key_len_percent_dist": "1,30,69",
"read_fault_one_in": lambda: random.choice([0, 1000]),
+ "open_metadata_write_fault_one_in": lambda: random.choice([0, 8]),
"sync_fault_injection": False,
"get_property_one_in": 1000000,
"paranoid_file_checks": lambda: random.choice([0, 1, 1, 1]),
if (!fs_->IsFilesystemActive()) {
return fs_->GetError();
}
+ {
+ IOStatus in_s = fs_->InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
fs_->SyncDir(dirname_);
- return dir_->Fsync(options, dbg);
+ IOStatus s = dir_->Fsync(options, dbg);
+ {
+ IOStatus in_s = fs_->InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
+ return s;
}
TestFSWritableFile::TestFSWritableFile(const std::string& fname,
if (!fs_->IsFilesystemActive()) {
return fs_->GetError();
}
+ {
+ IOStatus in_s = fs_->InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
writable_file_opened_ = false;
IOStatus io_s;
io_s = target_->Append(state_.buffer_, options, dbg);
}
if (io_s.ok()) {
fs_->WritableFileClosed(state_);
+ IOStatus in_s = fs_->InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
}
return io_s;
}
if (!IsFilesystemActive()) {
return GetError();
}
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
if (IsFilesystemDirectWritable()) {
return target()->NewWritableFile(fname, file_opts, result, dbg);
}
// WritableFileWriter* file is opened
// again then it will be truncated - so forget our saved state.
UntrackFile(fname);
- MutexLock l(&mutex_);
- open_files_.insert(fname);
- auto dir_and_name = TestFSGetDirAndName(fname);
- auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
- list.insert(dir_and_name.second);
+ {
+ MutexLock l(&mutex_);
+ open_files_.insert(fname);
+ auto dir_and_name = TestFSGetDirAndName(fname);
+ auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
+ list.insert(dir_and_name.second);
+ }
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
}
return io_s;
}
if (IsFilesystemDirectWritable()) {
return target()->ReopenWritableFile(fname, file_opts, result, dbg);
}
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
IOStatus io_s = target()->ReopenWritableFile(fname, file_opts, result, dbg);
if (io_s.ok()) {
result->reset(
// WritableFileWriter* file is opened
// again then it will be truncated - so forget our saved state.
UntrackFile(fname);
- MutexLock l(&mutex_);
- open_files_.insert(fname);
- auto dir_and_name = TestFSGetDirAndName(fname);
- auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
- list.insert(dir_and_name.second);
+ {
+ MutexLock l(&mutex_);
+ open_files_.insert(fname);
+ auto dir_and_name = TestFSGetDirAndName(fname);
+ auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
+ list.insert(dir_and_name.second);
+ }
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
}
return io_s;
}
if (IsFilesystemDirectWritable()) {
return target()->NewRandomRWFile(fname, file_opts, result, dbg);
}
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
IOStatus io_s = target()->NewRandomRWFile(fname, file_opts, result, dbg);
if (io_s.ok()) {
result->reset(new TestFSRandomRWFile(fname, std::move(*result), this));
// WritableFileWriter* file is opened
// again then it will be truncated - so forget our saved state.
UntrackFile(fname);
- MutexLock l(&mutex_);
- open_files_.insert(fname);
- auto dir_and_name = TestFSGetDirAndName(fname);
- auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
- list.insert(dir_and_name.second);
+ {
+ MutexLock l(&mutex_);
+ open_files_.insert(fname);
+ auto dir_and_name = TestFSGetDirAndName(fname);
+ auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
+ list.insert(dir_and_name.second);
+ }
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
}
return io_s;
}
if (!IsFilesystemActive()) {
return GetError();
}
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
IOStatus io_s = FileSystemWrapper::DeleteFile(f, options, dbg);
if (io_s.ok()) {
UntrackFile(f);
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
}
return io_s;
}
if (!IsFilesystemActive()) {
return GetError();
}
+ {
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
+ }
+ }
IOStatus io_s = FileSystemWrapper::RenameFile(s, t, options, dbg);
if (io_s.ok()) {
- MutexLock l(&mutex_);
- if (db_file_state_.find(s) != db_file_state_.end()) {
- db_file_state_[t] = db_file_state_[s];
- db_file_state_.erase(s);
- }
+ {
+ MutexLock l(&mutex_);
+ if (db_file_state_.find(s) != db_file_state_.end()) {
+ db_file_state_[t] = db_file_state_[s];
+ db_file_state_.erase(s);
+ }
- auto sdn = TestFSGetDirAndName(s);
- auto tdn = TestFSGetDirAndName(t);
- if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) {
- auto& tlist = dir_to_new_files_since_last_sync_[tdn.first];
- assert(tlist.find(tdn.second) == tlist.end());
- tlist.insert(tdn.second);
+ auto sdn = TestFSGetDirAndName(s);
+ auto tdn = TestFSGetDirAndName(t);
+ if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) {
+ auto& tlist = dir_to_new_files_since_last_sync_[tdn.first];
+ assert(tlist.find(tdn.second) == tlist.end());
+ tlist.insert(tdn.second);
+ }
+ }
+ IOStatus in_s = InjectMetadataWriteError();
+ if (!in_s.ok()) {
+ return in_s;
}
}
return IOStatus::OK();
}
+IOStatus FaultInjectionTestFS::InjectMetadataWriteError() {
+ MutexLock l(&mutex_);
+ if (!enable_metadata_write_error_injection_ ||
+ !metadata_write_error_one_in_ ||
+ !write_error_rand_.OneIn(metadata_write_error_one_in_)) {
+ return IOStatus::OK();
+ }
+ return IOStatus::IOError();
+}
+
void FaultInjectionTestFS::PrintFaultBacktrace() {
#if defined(OS_LINUX)
ErrorContext* ctx =
filesystem_writable_(false),
thread_local_error_(new ThreadLocalPtr(DeleteThreadLocalErrorContext)),
enable_write_error_injection_(false),
+ enable_metadata_write_error_injection_(false),
write_error_rand_(0),
+ write_error_one_in_(0),
+ metadata_write_error_one_in_(0),
ingest_data_corruption_before_write_(false) {}
virtual ~FaultInjectionTestFS() { error_.PermitUncheckedError(); }
write_error_allowed_types_ = types;
}
+ void SetRandomMetadataWriteError(int one_in) {
+ MutexLock l(&mutex_);
+ metadata_write_error_one_in_ = one_in;
+ }
+
// Inject an write error with randomlized parameter and the predefined
// error type. Only the allowed file types will inject the write error
IOStatus InjectWriteError(const std::string& file_name);
+ // Ingest error to metadata operations.
+ IOStatus InjectMetadataWriteError();
+
// Inject an error. For a READ operation, a status of IOError(), a
// corruption in the contents of scratch, or truncation of slice
// are the types of error with equal probability. For OPEN,
enable_write_error_injection_ = true;
}
+ void EnableMetadataWriteErrorInjection() {
+ MutexLock l(&mutex_);
+ enable_metadata_write_error_injection_ = true;
+ }
+
void DisableWriteErrorInjection() {
MutexLock l(&mutex_);
enable_write_error_injection_ = false;
}
}
+ void DisableMetadataWriteErrorInjection() {
+ MutexLock l(&mutex_);
+ enable_metadata_write_error_injection_ = false;
+ }
+
// We capture a backtrace every time a fault is injected, for debugging
// purposes. This call prints the backtrace to stderr and frees the
// saved callstack
std::unique_ptr<ThreadLocalPtr> thread_local_error_;
bool enable_write_error_injection_;
+ bool enable_metadata_write_error_injection_;
Random write_error_rand_;
int write_error_one_in_;
+ int metadata_write_error_one_in_;
std::vector<FileType> write_error_allowed_types_;
bool ingest_data_corruption_before_write_;
ChecksumType checksum_handoff_func_tpye_;
projects / rocksdb.git / commitdiff