diff --git a/db_stress_tool/db_stress_driver.cc b/db_stress_tool/db_stress_driver.cc index 12788ee5550d..6788741a96b1 100644 --- a/db_stress_tool/db_stress_driver.cc +++ b/db_stress_tool/db_stress_driver.cc @@ -159,13 +159,13 @@ bool RunStressTestImpl(SharedState* shared) { stress->TrackExpectedState(shared); } - // Since wrie fault and sync fault implementations are coupled with each - // other in `TestFSWritableFile()`, we can not enable or disable only one - // of the two. - // TODO(hx235): decouple implementations of write fault injection and sync - // fault injection. if (FLAGS_sync_fault_injection || FLAGS_write_fault_one_in > 0) { fault_fs_guard->SetFilesystemDirectWritable(false); + fault_fs_guard->SetInjectUnsyncedDataLoss(FLAGS_sync_fault_injection); + if (FLAGS_exclude_wal_from_write_fault_injection) { + fault_fs_guard->SetFileTypesExcludedFromFaultInjection( + {FileType::kWalFile}); + } } now = clock->NowMicros(); fprintf(stdout, "%s Starting database operations\n", diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 652973c30ba5..3fb2afa7caee 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -1095,6 +1095,9 @@ DEFINE_int32(write_fault_one_in, 0, "On non-zero, enables fault injection on write. Currently only" "injects write error when writing to SST files."); +DEFINE_bool(exclude_wal_from_write_fault_injection, false, + "If true, we won't inject write fault when writing to WAL file"); + DEFINE_int32(metadata_write_fault_one_in, 1000, "On non-zero, enables fault injection on metadata write (i.e, " "directory and file metadata write)"); diff --git a/db_stress_tool/db_stress_listener.h b/db_stress_tool/db_stress_listener.h index 85f232865cc8..5022b9775115 100644 --- a/db_stress_tool/db_stress_listener.h +++ b/db_stress_tool/db_stress_listener.h @@ -284,7 +284,7 @@ class DbStressListener : public EventListener { FaultInjectionIOType::kMetadataWrite); // TODO(hx235): only exempt the flush thread during error recovery instead // of all the flush threads from error injection - fault_fs_guard->SetIOActivtiesExemptedFromFaultInjection( + fault_fs_guard->SetIOActivtiesExcludedFromFaultInjection( {Env::IOActivity::kFlush}); } } @@ -300,7 +300,7 @@ class DbStressListener : public EventListener { FaultInjectionIOType::kMetadataRead); fault_fs_guard->EnableThreadLocalErrorInjection( FaultInjectionIOType::kMetadataWrite); - fault_fs_guard->SetIOActivtiesExemptedFromFaultInjection({}); + fault_fs_guard->SetIOActivtiesExcludedFromFaultInjection({}); } } diff --git a/db_stress_tool/db_stress_shared_state.h b/db_stress_tool/db_stress_shared_state.h index 6fad42f83c58..443243452bb7 100644 --- a/db_stress_tool/db_stress_shared_state.h +++ b/db_stress_tool/db_stress_shared_state.h @@ -37,6 +37,7 @@ DECLARE_int32(metadata_read_fault_one_in); DECLARE_int32(metadata_write_fault_one_in); DECLARE_int32(read_fault_one_in); DECLARE_int32(write_fault_one_in); +DECLARE_bool(exclude_wal_from_write_fault_injection); DECLARE_int32(open_metadata_read_fault_one_in); DECLARE_int32(open_metadata_write_fault_one_in); DECLARE_int32(open_write_fault_one_in); diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 19e3a6477985..17ba4396a1cd 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -436,7 +436,7 @@ void StressTest::FinishInitDb(SharedState* shared) { void StressTest::TrackExpectedState(SharedState* shared) { // When data loss is simulated, recovery from potential data loss is a prefix // recovery that requires tracing - if (MightHaveDataLoss() && IsStateTracked()) { + if (MightHaveUnsyncedDataLoss() && IsStateTracked()) { Status s = shared->SaveAtAndAfter(db_); if (!s.ok()) { fprintf(stderr, "Error enabling history tracing: %s\n", @@ -3426,17 +3426,23 @@ void StressTest::Open(SharedState* shared, bool reopen) { } // TODO; test transaction DB Open with fault injection if (!FLAGS_use_txn) { + bool inject_sync_fault = FLAGS_sync_fault_injection; bool inject_open_meta_read_error = FLAGS_open_metadata_read_fault_one_in > 0; bool inject_open_meta_write_error = FLAGS_open_metadata_write_fault_one_in > 0; bool inject_open_read_error = FLAGS_open_read_fault_one_in > 0; bool inject_open_write_error = FLAGS_open_write_fault_one_in > 0; - if ((inject_open_meta_read_error || inject_open_meta_write_error || - inject_open_read_error || inject_open_write_error) && + if ((inject_sync_fault || inject_open_meta_read_error || + inject_open_meta_write_error || inject_open_read_error || + inject_open_write_error) && fault_fs_guard ->FileExists(FLAGS_db + "/CURRENT", IOOptions(), nullptr) .ok()) { + if (inject_sync_fault || inject_open_write_error) { + fault_fs_guard->SetFilesystemDirectWritable(false); + fault_fs_guard->SetInjectUnsyncedDataLoss(inject_sync_fault); + } fault_fs_guard->SetThreadLocalErrorContext( FaultInjectionIOType::kMetadataRead, static_cast(FLAGS_seed), @@ -3460,7 +3466,6 @@ void StressTest::Open(SharedState* shared, bool reopen) { fault_fs_guard->EnableThreadLocalErrorInjection( FaultInjectionIOType::kRead); - fault_fs_guard->SetFilesystemDirectWritable(false); fault_fs_guard->SetThreadLocalErrorContext( FaultInjectionIOType::kWrite, static_cast(FLAGS_seed), FLAGS_open_write_fault_one_in, false /* retryable */, @@ -3495,11 +3500,12 @@ void StressTest::Open(SharedState* shared, bool reopen) { } } - if (inject_open_meta_read_error || inject_open_meta_write_error || - inject_open_read_error || inject_open_write_error) { + if (inject_sync_fault || inject_open_meta_read_error || + inject_open_meta_write_error || inject_open_read_error || + inject_open_write_error) { + fault_fs_guard->SetInjectUnsyncedDataLoss(false); fault_fs_guard->DisableThreadLocalErrorInjection( FaultInjectionIOType::kRead); - fault_fs_guard->SetFilesystemDirectWritable(true); fault_fs_guard->DisableThreadLocalErrorInjection( FaultInjectionIOType::kWrite); fault_fs_guard->DisableThreadLocalErrorInjection( @@ -3522,9 +3528,10 @@ void StressTest::Open(SharedState* shared, bool reopen) { } } if (!s.ok()) { - // After failure to opening a DB due to IO error, retry should - // successfully open the DB with correct data if no IO error shows - // up. + // After failure to opening a DB due to IO error or unsynced data + // loss, retry should successfully open the DB with correct data if + // no IO error shows up. + inject_sync_fault = false; inject_open_meta_read_error = false; inject_open_meta_write_error = false; inject_open_read_error = false; @@ -3721,7 +3728,7 @@ void StressTest::Reopen(ThreadState* thread) { clock_->TimeToString(now / 1000000).c_str(), num_times_reopened_); Open(thread->shared, /*reopen=*/true); - if (thread->shared->GetStressTest()->MightHaveDataLoss() && + if (thread->shared->GetStressTest()->MightHaveUnsyncedDataLoss() && IsStateTracked()) { Status s = thread->shared->SaveAtAndAfter(db_); if (!s.ok()) { diff --git a/db_stress_tool/db_stress_test_base.h b/db_stress_tool/db_stress_test_base.h index 213d9a4101de..de17ecd0652b 100644 --- a/db_stress_tool/db_stress_test_base.h +++ b/db_stress_tool/db_stress_test_base.h @@ -44,9 +44,8 @@ class StressTest { virtual void VerifyDb(ThreadState* thread) const = 0; virtual void ContinuouslyVerifyDb(ThreadState* /*thread*/) const = 0; void PrintStatistics(); - bool MightHaveDataLoss() { - return FLAGS_sync_fault_injection || FLAGS_write_fault_one_in > 0 || - FLAGS_metadata_write_fault_one_in > 0 || FLAGS_disable_wal || + bool MightHaveUnsyncedDataLoss() { + return FLAGS_sync_fault_injection || FLAGS_disable_wal || FLAGS_manual_wal_flush_one_in > 0; } diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index 8cc459f12a64..f424f419f42e 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -90,10 +90,10 @@ int db_stress_tool(int argc, char** argv) { FaultInjectionTestFS* fs = new FaultInjectionTestFS(raw_env->GetFileSystem()); fault_fs_guard.reset(fs); - // Set it to direct writable here to not lose files created during DB open - // when no open fault injection is not enabled. - // This will be overwritten in StressTest::Open() for open fault injection - // and in RunStressTestImpl() for proper write fault injection setup. + // Set it to direct writable here to initially bypass any fault injection + // during DB open This will correspondingly be overwritten in + // StressTest::Open() for open fault injection and in RunStressTestImpl() + // for proper fault injection setup. fault_fs_guard->SetFilesystemDirectWritable(true); fault_env_guard = std::make_shared(raw_env, fault_fs_guard); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 948b2c2342c1..1ff2ebe6d35c 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -222,6 +222,7 @@ "metadata_write_fault_one_in": lambda: random.choice([0, 128, 1000]), "read_fault_one_in": lambda: random.choice([0, 32, 1000]), "write_fault_one_in": lambda: random.choice([0, 128, 1000]), + "exclude_wal_from_write_fault_injection": 0, "open_metadata_write_fault_one_in": lambda: random.choice([0, 0, 8]), "open_metadata_read_fault_one_in": lambda: random.choice([0, 0, 8]), "open_write_fault_one_in": lambda: random.choice([0, 0, 16]), @@ -622,11 +623,11 @@ def is_direct_io_supported(dbname): "enable_compaction_filter": 0, "create_timestamped_snapshot_one_in": 50, "sync_fault_injection": 0, - "metadata_write_fault_one_in": 0, "manual_wal_flush": 0, # This test has aggressive flush frequency and small write buffer size. # Disabling write fault to avoid writes being stopped. "write_fault_one_in": 0, + "metadata_write_fault_one_in": 0, # PutEntity in transactions is not yet implemented "use_put_entity_one_in": 0, "use_get_entity": 0, @@ -736,14 +737,10 @@ def finalize_and_sanitize(src_params): # logic for unsynced data loss relies on max sequence number stored # in MANIFEST, so they don't work together. dest_params["sync_fault_injection"] = 0 - dest_params["write_fault_one_in"] = 0 - dest_params["metadata_write_fault_one_in"] = 0 dest_params["disable_wal"] = 0 dest_params["manual_wal_flush_one_in"] = 0 if ( dest_params.get("sync_fault_injection") == 1 - or dest_params.get("write_fault_one_in") > 0 - or dest_params.get("metadata_write_fault_one_in") > 0 or dest_params.get("disable_wal") == 1 or dest_params.get("manual_wal_flush_one_in") > 0 ): @@ -759,6 +756,11 @@ def finalize_and_sanitize(src_params): # files, which would be problematic when unsynced data can be lost in # crash recoveries. dest_params["enable_compaction_filter"] = 0 + # Prefix-recoverability relies on tracing successful user writes. Currently we trace all user writes regardless of whether it later succeeds or not. + # To simplify, we disable any user write failure injection. + # TODO(hx235): support tracing user writes with failure injection. + dest_params["metadata_write_fault_one_in"] = 0 + dest_params["exclude_wal_from_write_fault_injection"] = 1 # Only under WritePrepared txns, unordered_write would provide the same guarnatees as vanilla rocksdb # unordered_write is only enabled with --txn, and txn_params disables inplace_update_support, so # setting allow_concurrent_memtable_write=1 won't conflcit with inplace_update_support. @@ -813,8 +815,6 @@ def finalize_and_sanitize(src_params): # compatible with only write committed policy if dest_params.get("use_txn") == 1 and dest_params.get("txn_write_policy", 0) != 0: dest_params["sync_fault_injection"] = 0 - dest_params["write_fault_one_in"] = 0 - dest_params["metadata_write_fault_one_in"] = 0 dest_params["disable_wal"] = 0 dest_params["manual_wal_flush_one_in"] = 0 # Wide-column pessimistic transaction APIs are initially supported for diff --git a/utilities/fault_injection_fs.cc b/utilities/fault_injection_fs.cc index ba451adc0245..9e1b9f4e5a38 100644 --- a/utilities/fault_injection_fs.cc +++ b/utilities/fault_injection_fs.cc @@ -166,13 +166,13 @@ IOStatus TestFSWritableFile::Append(const Slice& data, const IOOptions& options, return fs_->GetError(); } - IOStatus s = - fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, options); + IOStatus s = fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + options, state_.filename_); if (!s.ok()) { return s; } - if (target_->use_direct_io()) { + if (target_->use_direct_io() || !fs_->InjectUnsyncedDataLoss()) { // TODO(hx235): buffer data for direct IO write to simulate data loss like // non-direct IO write s = target_->Append(data, options, dbg); @@ -201,8 +201,8 @@ IOStatus TestFSWritableFile::Append( return IOStatus::Corruption("Data is corrupted!"); } - IOStatus s = - fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, options); + IOStatus s = fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + options, state_.filename_); if (!s.ok()) { return s; } @@ -220,7 +220,7 @@ IOStatus TestFSWritableFile::Append( return IOStatus::Corruption(msg); } - if (target_->use_direct_io()) { + if (target_->use_direct_io() || !fs_->InjectUnsyncedDataLoss()) { // TODO(hx235): buffer data for direct IO write to simulate data loss like // non-direct IO write s = target_->Append(data, options, dbg); @@ -240,8 +240,8 @@ IOStatus TestFSWritableFile::Truncate(uint64_t size, const IOOptions& options, if (!fs_->IsFilesystemActive()) { return fs_->GetError(); } - IOStatus s = - fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, options); + IOStatus s = fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + options, state_.filename_); if (!s.ok()) { return s; } @@ -264,8 +264,8 @@ IOStatus TestFSWritableFile::PositionedAppend(const Slice& data, if (fs_->ShouldDataCorruptionBeforeWrite()) { return IOStatus::Corruption("Data is corrupted!"); } - IOStatus s = - fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, options); + IOStatus s = fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + options, state_.filename_); if (!s.ok()) { return s; } @@ -290,8 +290,8 @@ IOStatus TestFSWritableFile::PositionedAppend( if (fs_->ShouldDataCorruptionBeforeWrite()) { return IOStatus::Corruption("Data is corrupted!"); } - IOStatus s = - fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, options); + IOStatus s = fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + options, state_.filename_); if (!s.ok()) { return s; } @@ -468,7 +468,7 @@ IOStatus TestFSRandomAccessFile::Read(uint64_t offset, size_t n, return fs_->GetError(); } IOStatus s = fs_->MaybeInjectThreadLocalError( - FaultInjectionIOType::kRead, options, + FaultInjectionIOType::kRead, options, "", FaultInjectionTestFS::ErrorOperation::kRead, result, use_direct_io(), scratch, /*need_count_increase=*/true, /*fault_injected=*/nullptr); @@ -492,7 +492,7 @@ IOStatus TestFSRandomAccessFile::ReadAsync( } if (s.ok()) { s = fs_->MaybeInjectThreadLocalError( - FaultInjectionIOType::kRead, opts, + FaultInjectionIOType::kRead, opts, "", FaultInjectionTestFS::ErrorOperation::kRead, &res.result, use_direct_io(), req.scratch, /*need_count_increase=*/true, /*fault_injected=*/nullptr); @@ -524,7 +524,7 @@ IOStatus TestFSRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs, } bool this_injected_error; reqs[i].status = fs_->MaybeInjectThreadLocalError( - FaultInjectionIOType::kRead, options, + FaultInjectionIOType::kRead, options, "", FaultInjectionTestFS::ErrorOperation::kRead, &(reqs[i].result), use_direct_io(), reqs[i].scratch, /*need_count_increase=*/true, @@ -533,7 +533,7 @@ IOStatus TestFSRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs, } if (s.ok()) { s = fs_->MaybeInjectThreadLocalError( - FaultInjectionIOType::kRead, options, + FaultInjectionIOType::kRead, options, "", FaultInjectionTestFS::ErrorOperation::kMultiRead, nullptr, use_direct_io(), nullptr, /*need_count_increase=*/!injected_error, /*fault_injected=*/nullptr); @@ -589,7 +589,7 @@ IOStatus TestFSSequentialFile::Read(size_t n, const IOOptions& options, Slice* result, char* scratch, IODebugContext* dbg) { IOStatus s = fs_->MaybeInjectThreadLocalError( - FaultInjectionIOType::kRead, options, + FaultInjectionIOType::kRead, options, "", FaultInjectionTestFS::ErrorOperation::kRead, result, use_direct_io(), scratch, true /*need_count_increase=*/, nullptr /* fault_injected*/); if (!s.ok()) { @@ -614,7 +614,7 @@ IOStatus TestFSSequentialFile::PositionedRead(uint64_t offset, size_t n, Slice* result, char* scratch, IODebugContext* dbg) { IOStatus s = fs_->MaybeInjectThreadLocalError( - FaultInjectionIOType::kRead, options, + FaultInjectionIOType::kRead, options, "", FaultInjectionTestFS::ErrorOperation::kRead, result, use_direct_io(), scratch, true /*need_count_increase=*/, nullptr /* fault_injected */); if (!s.ok()) { @@ -698,12 +698,12 @@ IOStatus FaultInjectionTestFS::NewWritableFile( return GetError(); } - if (ShouldUseDiretWritable(fname)) { + if (IsFilesystemDirectWritable()) { return target()->NewWritableFile(fname, file_opts, result, dbg); } - IOStatus io_s = MaybeInjectThreadLocalError( - FaultInjectionIOType::kMetadataWrite, file_opts.io_options); + IOStatus io_s = MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + file_opts.io_options, fname); if (!io_s.ok()) { return io_s; } @@ -735,11 +735,11 @@ IOStatus FaultInjectionTestFS::ReopenWritableFile( if (!IsFilesystemActive()) { return GetError(); } - if (ShouldUseDiretWritable(fname)) { + if (IsFilesystemDirectWritable()) { return target()->ReopenWritableFile(fname, file_opts, result, dbg); } - IOStatus io_s = MaybeInjectThreadLocalError( - FaultInjectionIOType::kMetadataWrite, file_opts.io_options); + IOStatus io_s = MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + file_opts.io_options, fname); if (!io_s.ok()) { return io_s; } @@ -811,11 +811,11 @@ IOStatus FaultInjectionTestFS::NewRandomRWFile( if (!IsFilesystemActive()) { return GetError(); } - if (ShouldUseDiretWritable(fname)) { + if (IsFilesystemDirectWritable()) { return target()->NewRandomRWFile(fname, file_opts, result, dbg); } - IOStatus io_s = MaybeInjectThreadLocalError( - FaultInjectionIOType::kMetadataWrite, file_opts.io_options); + IOStatus io_s = MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite, + file_opts.io_options, fname); if (!io_s.ok()) { return io_s; } @@ -847,7 +847,10 @@ IOStatus FaultInjectionTestFS::NewRandomAccessFile( return GetError(); } IOStatus io_s = MaybeInjectThreadLocalError( - FaultInjectionIOType::kMetadataWrite, file_opts.io_options); + FaultInjectionIOType::kRead, file_opts.io_options, fname, + ErrorOperation::kOpen, nullptr /* result */, false /* direct_io */, + nullptr /* scratch */, true /*need_count_increase*/, + nullptr /*fault_injected*/); if (!io_s.ok()) { return io_s; } @@ -867,7 +870,10 @@ IOStatus FaultInjectionTestFS::NewSequentialFile( return GetError(); } IOStatus io_s = MaybeInjectThreadLocalError( - FaultInjectionIOType::kMetadataWrite, file_opts.io_options); + FaultInjectionIOType::kRead, file_opts.io_options, fname, + ErrorOperation::kOpen, nullptr /* result */, false /* direct_io */, + nullptr /* scratch */, true /*need_count_increase*/, + nullptr /*fault_injected*/); if (!io_s.ok()) { return io_s; } @@ -1228,8 +1234,8 @@ IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalReadError( ErrorContext* ctx = static_cast(injected_thread_local_read_error_.Get()); if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in || - io_activties_exempted_from_fault_injection.find(io_options.io_activity) != - io_activties_exempted_from_fault_injection.end()) { + io_activties_excluded_from_fault_injection.find(io_options.io_activity) != + io_activties_excluded_from_fault_injection.end()) { return IOStatus::OK(); } @@ -1295,8 +1301,9 @@ bool FaultInjectionTestFS::TryParseFileName(const std::string& file_name, } IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalError( - FaultInjectionIOType type, const IOOptions& io_options, ErrorOperation op, - Slice* result, bool direct_io, char* scratch, bool need_count_increase, + FaultInjectionIOType type, const IOOptions& io_options, + const std::string& file_name, ErrorOperation op, Slice* result, + bool direct_io, char* scratch, bool need_count_increase, bool* fault_injected) { if (type == FaultInjectionIOType::kRead) { return MaybeInjectThreadLocalReadError(io_options, op, result, direct_io, @@ -1306,8 +1313,10 @@ IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalError( ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type); if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in || - io_activties_exempted_from_fault_injection.find(io_options.io_activity) != - io_activties_exempted_from_fault_injection.end()) { + io_activties_excluded_from_fault_injection.find(io_options.io_activity) != + io_activties_excluded_from_fault_injection.end() || + (type == FaultInjectionIOType::kWrite && + ShouldExcludeFromWriteInjection(file_name))) { return IOStatus::OK(); } diff --git a/utilities/fault_injection_fs.h b/utilities/fault_injection_fs.h index 5b4916066584..3dadf86971f9 100644 --- a/utilities/fault_injection_fs.h +++ b/utilities/fault_injection_fs.h @@ -207,6 +207,7 @@ class FaultInjectionTestFS : public FileSystemWrapper { : FileSystemWrapper(base), filesystem_active_(true), filesystem_writable_(false), + inject_unsynced_data_loss_(false), read_unsynced_data_(true), allow_link_open_file_(false), injected_thread_local_read_error_(DeleteThreadLocalErrorContext), @@ -359,19 +360,6 @@ class FaultInjectionTestFS : public FileSystemWrapper { MutexLock l(&mutex_); return filesystem_writable_; } - bool ShouldUseDiretWritable(const std::string& file_name) { - MutexLock l(&mutex_); - if (filesystem_writable_) { - return true; - } - FileType file_type = kTempFile; - uint64_t file_number = 0; - if (!TryParseFileName(file_name, &file_number, &file_type)) { - return false; - } - return direct_writable_types_.find(file_type) != - direct_writable_types_.end(); - } void SetFilesystemActiveNoLock( bool active, IOStatus error = IOStatus::Corruption("Not active")) { error.PermitUncheckedError(); @@ -391,6 +379,18 @@ class FaultInjectionTestFS : public FileSystemWrapper { filesystem_writable_ = writable; } + // If true, we buffer write data in memory to simulate data loss upon system + // crash by only having process crashes + void SetInjectUnsyncedDataLoss(bool inject) { + MutexLock l(&mutex_); + inject_unsynced_data_loss_ = inject; + } + + bool InjectUnsyncedDataLoss() { + MutexLock l(&mutex_); + return inject_unsynced_data_loss_; + } + // In places (e.g. GetSortedWals()) RocksDB relies on querying the file size // or even reading the contents of files currently open for writing, and // as in POSIX semantics, expects to see the flushed size and contents @@ -414,16 +414,6 @@ class FaultInjectionTestFS : public FileSystemWrapper { allow_link_open_file_ = allow_link_open_file; } - void SetDirectWritableTypes(const std::set& types) { - MutexLock l(&mutex_); - direct_writable_types_ = types; - } - - void SetIOActivtiesExemptedFromFaultInjection( - const std::set& io_activties) { - MutexLock l(&mutex_); - io_activties_exempted_from_fault_injection = io_activties; - } void AssertNoOpenFile() { assert(open_managed_files_.empty()); } @@ -499,8 +489,8 @@ class FaultInjectionTestFS : public FileSystemWrapper { IOStatus MaybeInjectThreadLocalError( FaultInjectionIOType type, const IOOptions& io_options, - ErrorOperation op = kUnknown, Slice* slice = nullptr, - bool direct_io = false, char* scratch = nullptr, + const std::string& file_name = "", ErrorOperation op = kUnknown, + Slice* slice = nullptr, bool direct_io = false, char* scratch = nullptr, bool need_count_increase = false, bool* fault_injected = nullptr); int GetAndResetInjectedThreadLocalErrorCount(FaultInjectionIOType type) { @@ -513,6 +503,28 @@ class FaultInjectionTestFS : public FileSystemWrapper { return count; } + void SetIOActivtiesExcludedFromFaultInjection( + const std::set& io_activties) { + MutexLock l(&mutex_); + io_activties_excluded_from_fault_injection = io_activties; + } + + void SetFileTypesExcludedFromFaultInjection(const std::set& types) { + MutexLock l(&mutex_); + file_types_excluded_from_write_fault_injection_ = types; + } + + bool ShouldExcludeFromWriteInjection(const std::string& file_name) { + MutexLock l(&mutex_); + FileType file_type = kTempFile; + uint64_t file_number = 0; + if (!TryParseFileName(file_name, &file_number, &file_type)) { + return false; + } + return file_types_excluded_from_write_fault_injection_.find(file_type) != + file_types_excluded_from_write_fault_injection_.end(); + } + void EnableThreadLocalErrorInjection(FaultInjectionIOType type) { ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type); if (ctx) { @@ -545,6 +557,7 @@ class FaultInjectionTestFS : public FileSystemWrapper { bool filesystem_active_; // Record flushes, syncs, writes bool filesystem_writable_; // Bypass FaultInjectionTestFS and go directly // to underlying FS for writable files + bool inject_unsynced_data_loss_; // See InjectUnsyncedDataLoss() bool read_unsynced_data_; // See SetReadUnsyncedData() bool allow_link_open_file_; // See SetAllowLinkOpenFile() IOStatus fs_error_; @@ -582,8 +595,8 @@ class FaultInjectionTestFS : public FileSystemWrapper { } }; - std::set direct_writable_types_; - std::set io_activties_exempted_from_fault_injection; + std::set file_types_excluded_from_write_fault_injection_; + std::set io_activties_excluded_from_fault_injection; ThreadLocalPtr injected_thread_local_read_error_; ThreadLocalPtr injected_thread_local_write_error_; ThreadLocalPtr injected_thread_local_metadata_read_error_;