Skip to content

Commit

Permalink
Fix not getting expected injected read error (facebook#12793)
Browse files Browse the repository at this point in the history 
Summary:
**Context/Summary:**

facebook#12713 accidentally removed the mechanism of ignoring injected read error on non-critical read path such as read from filter. IO failure in read from filter should not fail the read as we can always read from the actual file. Therefore error injection in filter read path does not need to lead to failure in Get() and crash test should allow that. Otherwise, we will get crash test error "Didn't get expected error from..."

Pull Request resolved: facebook#12793

Test Plan: CI

Reviewed By: cbi42

Differential Revision: D58895393

Pulled By: hx235

fbshipit-source-id: 5b605d8446e0b8d4149cdbe6f4be3c7534d4acfa
  • Loading branch information
@hx235 @facebook-github-bot
hx235 authored and facebook-github-bot committed Jun 22, 2024
1 parent d6cf9de commit 981fd43
Showing 1 changed file with 12 additions and 6 deletions.
18 changes: 12 additions & 6 deletions db_stress_tool/no_batched_ops_stress.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,7 @@ class NonBatchedOpsStressTest : public StressTest {
FaultInjectionIOType::kRead);
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kMetadataRead);
SharedState::ignore_read_error = false;
}

const ExpectedValue pre_read_expected_value =
Expand All @@ -537,7 +538,7 @@ class NonBatchedOpsStressTest : public StressTest {
thread->shared->Get(rand_column_families[0], rand_keys[0]);

int injected_error_count = 0;
if (fault_fs_guard) {
if (fault_fs_guard && !SharedState::ignore_read_error) {
injected_error_count = GetMinInjectedErrorCount(
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kRead),
Expand Down Expand Up @@ -687,10 +688,11 @@ class NonBatchedOpsStressTest : public StressTest {
FaultInjectionIOType::kRead);
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kMetadataRead);
SharedState::ignore_read_error = false;
}
db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(),
statuses.data());
if (fault_fs_guard) {
if (fault_fs_guard && !SharedState::ignore_read_error) {
injected_error_count = GetMinInjectedErrorCount(
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kRead),
Expand Down Expand Up @@ -981,6 +983,7 @@ class NonBatchedOpsStressTest : public StressTest {
FaultInjectionIOType::kRead);
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kMetadataRead);
SharedState::ignore_read_error = false;
}

Status s;
Expand All @@ -998,7 +1001,7 @@ class NonBatchedOpsStressTest : public StressTest {
thread->shared->Get(column_family, key);

int injected_error_count = 0;
if (fault_fs_guard) {
if (fault_fs_guard && !SharedState::ignore_read_error) {
injected_error_count = GetMinInjectedErrorCount(
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kRead),
Expand Down Expand Up @@ -1384,6 +1387,7 @@ class NonBatchedOpsStressTest : public StressTest {
FaultInjectionIOType::kRead);
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kMetadataRead);
SharedState::ignore_read_error = false;
}

std::vector<PinnableAttributeGroups> results;
Expand All @@ -1397,7 +1401,7 @@ class NonBatchedOpsStressTest : public StressTest {
db_->MultiGetEntity(read_opts_copy, num_keys, key_slices.data(),
results.data());

if (fault_fs_guard) {
if (fault_fs_guard && !SharedState::ignore_read_error) {
verify_expected_errors(
[&](size_t i) { return results[i][0].status(); });
}
Expand All @@ -1418,6 +1422,7 @@ class NonBatchedOpsStressTest : public StressTest {
FaultInjectionIOType::kRead);
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kMetadataRead);
SharedState::ignore_read_error = false;
}

std::vector<PinnableWideColumns> results(num_keys);
Expand All @@ -1426,7 +1431,7 @@ class NonBatchedOpsStressTest : public StressTest {
db_->MultiGetEntity(read_opts_copy, cfh, num_keys, key_slices.data(),
results.data(), statuses.data());

if (fault_fs_guard) {
if (fault_fs_guard && !SharedState::ignore_read_error) {
verify_expected_errors([&](size_t i) { return statuses[i]; });
}

Expand Down Expand Up @@ -1483,6 +1488,7 @@ class NonBatchedOpsStressTest : public StressTest {
FaultInjectionIOType::kRead);
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kMetadataRead);
SharedState::ignore_read_error = false;
}

for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix);
Expand Down Expand Up @@ -1516,7 +1522,7 @@ class NonBatchedOpsStressTest : public StressTest {
}

int injected_error_count = 0;
if (fault_fs_guard) {
if (fault_fs_guard && !SharedState::ignore_read_error) {
injected_error_count = GetMinInjectedErrorCount(
fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
FaultInjectionIOType::kRead),
Expand Down

0 comments on commit 981fd43

Please sign in to comment.