fix

db_stress_tool/db_stress_shared_state.h

@@ -262,14 +262,10 @@ class SharedState {
   // This is useful for crash-recovery testing when the process may crash
   // before updating the corresponding expected value
   //
-  // It can fail and `*prepared` will be set to false if the previous write or
-  // delete is still in pending state (e.g, still in recovery for retryable IO
-  // errors). If succeeds,`*prepared` will be set to true
-  //
   // Requires external locking covering `key` in `cf` to prevent
   // concurrent write or delete to the same `key`.
-  PendingExpectedValue PreparePut(int cf, int64_t key, bool* prepared) {
-    return expected_state_manager_->PreparePut(cf, key, prepared);
+  PendingExpectedValue PreparePut(int cf, int64_t key) {
+    return expected_state_manager_->PreparePut(cf, key);
   }
 
   // Does not requires external locking.
@@ -281,31 +277,24 @@ class SharedState {
   // This is useful for crash-recovery testing when the process may crash
   // before updating the corresponding expected value
   //
-  // It can fail and `*prepared` will be set to false if the previous write or
-  // delete is still in pending state (e.g, still in recovery for retryable IO
-  // errors). If succeeds,`*prepared` will be set to true
-  //
   // Requires external locking covering `key` in `cf` to prevent concurrent
   // write or delete to the same `key`.
-  PendingExpectedValue PrepareDelete(int cf, int64_t key, bool* prepared) {
-    return expected_state_manager_->PrepareDelete(cf, key, prepared);
+  PendingExpectedValue PrepareDelete(int cf, int64_t key) {
+    return expected_state_manager_->PrepareDelete(cf, key);
   }
 
   // Requires external locking covering `key` in `cf` to prevent concurrent
   // write or delete to the same `key`.
-  PendingExpectedValue PrepareSingleDelete(int cf, int64_t key,
-                                           bool* prepared) {
-    return expected_state_manager_->PrepareSingleDelete(cf, key, prepared);
+  PendingExpectedValue PrepareSingleDelete(int cf, int64_t key) {
+    return expected_state_manager_->PrepareSingleDelete(cf, key);
   }
 
   // Requires external locking covering keys in `[begin_key, end_key)` in `cf`
   // to prevent concurrent write or delete to the same `key`.
   std::vector<PendingExpectedValue> PrepareDeleteRange(int cf,
                                                        int64_t begin_key,
-                                                       int64_t end_key,
-                                                       bool* prepared) {
-    return expected_state_manager_->PrepareDeleteRange(cf, begin_key, end_key,
-                                                       prepared);
+                                                       int64_t end_key) {
+    return expected_state_manager_->PrepareDeleteRange(cf, begin_key, end_key);
   }
 
   bool AllowsOverwrite(int64_t key) const {

db_stress_tool/db_stress_test_base.cc

@@ -632,10 +632,8 @@ void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys,
   for (auto cfh : column_families_) {
     for (int64_t k = 0; k != number_of_keys; ++k) {
       const std::string key = Key(k);
-      bool prepare = false;
       PendingExpectedValue pending_expected_value =
-          shared->PreparePut(cf_idx, k, &prepare);
-      assert(prepare);
+          shared->PreparePut(cf_idx, k);
       const uint32_t value_base = pending_expected_value.GetFinalValueBase();
       const size_t sz = GenerateValue(value_base, value, sizeof(value));
 

db_stress_tool/db_stress_test_base.h

@@ -64,6 +64,42 @@ class StressTest {
     }
   }
 
+  void UpdateIfInitialWriteFails(Env* db_stress_env, const Status& write_s,
+                                 Status* initial_write_s,
+                                 bool* initial_wal_write_may_succeed,
+                                 uint64_t* wait_for_recover_start_time) {
+    assert(db_stress_env && initial_write_s && initial_wal_write_may_succeed &&
+           wait_for_recover_start_time);
+    // Only update `initial_write_s`, `initial_wal_write_may_succeed` when the
+    // first write fails
+    if (!write_s.ok() && (*initial_write_s).ok()) {
+      *initial_write_s = write_s;
+      *initial_wal_write_may_succeed =
+          !FaultInjectionTestFS::IsFailedToWriteToWALError(*initial_write_s);
+      *wait_for_recover_start_time = db_stress_env->NowMicros();
+    }
+  }
+
+  void PrintWriteRecoveryWaitTimeIfNeeded(Env* db_stress_env,
+                                          const Status& initial_write_s,
+                                          bool initial_wal_write_may_succeed,
+                                          uint64_t wait_for_recover_start_time,
+                                          const std::string& thread_name) {
+    assert(db_stress_env);
+    bool waited_for_recovery = !initial_write_s.ok() &&
+                               IsErrorInjectedAndRetryable(initial_write_s) &&
+                               initial_wal_write_may_succeed;
+    if (waited_for_recovery) {
+      uint64_t elapsed_sec =
+          (db_stress_env->NowMicros() - wait_for_recover_start_time) / 1000000;
+      if (elapsed_sec > 10) {
+        fprintf(stdout,
+                "%s thread slept to wait for write recovery for "
+                "%" PRIu64 " seconds\n",
+                thread_name.c_str(), elapsed_sec);
+      }
+    }
+  }
   void GetDeleteRangeKeyLocks(
       ThreadState* thread, int rand_column_family, int64_t rand_key,
       std::vector<std::unique_ptr<MutexLock>>* range_locks) {

db_stress_tool/expected_state.cc

@@ -32,41 +32,29 @@ void ExpectedState::Precommit(int cf, int64_t key, const ExpectedValue& value) {
   std::atomic_thread_fence(std::memory_order_release);
 }
 
-PendingExpectedValue ExpectedState::PreparePut(int cf, int64_t key,
-                                               bool* prepared) {
-  assert(prepared);
+PendingExpectedValue ExpectedState::PreparePut(int cf, int64_t key) {
   ExpectedValue expected_value = Load(cf, key);
 
   // Calculate the original expected value
   const ExpectedValue orig_expected_value = expected_value;
 
   // Calculate the pending expected value
-  bool res = expected_value.Put(true /* pending */);
-  if (!res) {
-    PendingExpectedValue ret = PendingExpectedValue(
-        &Value(cf, key), orig_expected_value, orig_expected_value);
-    *prepared = false;
-    return ret;
-  }
+  expected_value.Put(true /* pending */);
   const ExpectedValue pending_expected_value = expected_value;
 
   // Calculate the final expected value
-  res = expected_value.Put(false /* pending */);
-  assert(res);
+  expected_value.Put(false /* pending */);
   const ExpectedValue final_expected_value = expected_value;
 
   // Precommit
   Precommit(cf, key, pending_expected_value);
-  *prepared = true;
   return PendingExpectedValue(&Value(cf, key), orig_expected_value,
                               final_expected_value);
 }
 
 ExpectedValue ExpectedState::Get(int cf, int64_t key) { return Load(cf, key); }
 
-PendingExpectedValue ExpectedState::PrepareDelete(int cf, int64_t key,
-                                                  bool* prepared) {
-  assert(prepared);
+PendingExpectedValue ExpectedState::PrepareDelete(int cf, int64_t key) {
   ExpectedValue expected_value = Load(cf, key);
 
   // Calculate the original expected value
@@ -77,47 +65,32 @@ PendingExpectedValue ExpectedState::PrepareDelete(int cf, int64_t key,
   if (!res) {
     PendingExpectedValue ret = PendingExpectedValue(
         &Value(cf, key), orig_expected_value, orig_expected_value);
-    *prepared = false;
     return ret;
   }
   const ExpectedValue pending_expected_value = expected_value;
 
   // Calculate the final expected value
-  res = expected_value.Delete(false /* pending */);
-  assert(res);
+  expected_value.Delete(false /* pending */);
   const ExpectedValue final_expected_value = expected_value;
 
   // Precommit
   Precommit(cf, key, pending_expected_value);
-  *prepared = true;
   return PendingExpectedValue(&Value(cf, key), orig_expected_value,
                               final_expected_value);
 }
 
-PendingExpectedValue ExpectedState::PrepareSingleDelete(int cf, int64_t key,
-                                                        bool* prepared) {
-  return PrepareDelete(cf, key, prepared);
+PendingExpectedValue ExpectedState::PrepareSingleDelete(int cf, int64_t key) {
+  return PrepareDelete(cf, key);
 }
 
 std::vector<PendingExpectedValue> ExpectedState::PrepareDeleteRange(
-    int cf, int64_t begin_key, int64_t end_key, bool* prepared) {
+    int cf, int64_t begin_key, int64_t end_key) {
   std::vector<PendingExpectedValue> pending_expected_values;
-  bool has_prepared_failed = false;
 
   for (int64_t key = begin_key; key < end_key; ++key) {
-    bool each_prepared = false;
-    PendingExpectedValue pending_expected_value =
-        PrepareDelete(cf, key, &each_prepared);
-    if (each_prepared) {
-      pending_expected_values.push_back(pending_expected_value);
-    } else {
-      has_prepared_failed = true;
-      pending_expected_value.PermitUnclosedPendingState();
-      break;
-    }
+    pending_expected_values.push_back(PrepareDelete(cf, key));
   }
 
-  *prepared = !has_prepared_failed;
   return pending_expected_values;
 }
 

db_stress_tool/expected_state.h

@@ -44,7 +44,7 @@ class ExpectedState {
   //
   // Requires external locking covering `key` in `cf` to prevent concurrent
   // write or delete to the same `key`.
-  PendingExpectedValue PreparePut(int cf, int64_t key, bool* prepared);
+  PendingExpectedValue PreparePut(int cf, int64_t key);
 
   // Does not requires external locking.
   ExpectedValue Get(int cf, int64_t key);
@@ -55,18 +55,17 @@ class ExpectedState {
   //
   // Requires external locking covering `key` in `cf` to prevent concurrent
   // write or delete to the same `key`.
-  PendingExpectedValue PrepareDelete(int cf, int64_t key, bool* prepared);
+  PendingExpectedValue PrepareDelete(int cf, int64_t key);
 
   // Requires external locking covering `key` in `cf` to prevent concurrent
   // write or delete to the same `key`.
-  PendingExpectedValue PrepareSingleDelete(int cf, int64_t key, bool* prepared);
+  PendingExpectedValue PrepareSingleDelete(int cf, int64_t key);
 
   // Requires external locking covering keys in `[begin_key, end_key)` in `cf`
   // to prevent concurrent write or delete to the same `key`.
   std::vector<PendingExpectedValue> PrepareDeleteRange(int cf,
                                                        int64_t begin_key,
-                                                       int64_t end_key,
-                                                       bool* prepared);
+                                                       int64_t end_key);
 
   // Update the expected value for start of an incomplete write or delete
   // operation on the key assoicated with this expected value
@@ -197,30 +196,28 @@ class ExpectedStateManager {
   void ClearColumnFamily(int cf) { return latest_->ClearColumnFamily(cf); }
 
   // See ExpectedState::PreparePut()
-  PendingExpectedValue PreparePut(int cf, int64_t key, bool* prepared) {
-    return latest_->PreparePut(cf, key, prepared);
+  PendingExpectedValue PreparePut(int cf, int64_t key) {
+    return latest_->PreparePut(cf, key);
   }
 
   // See ExpectedState::Get()
   ExpectedValue Get(int cf, int64_t key) { return latest_->Get(cf, key); }
 
   // See ExpectedState::PrepareDelete()
-  PendingExpectedValue PrepareDelete(int cf, int64_t key, bool* prepared) {
-    return latest_->PrepareDelete(cf, key, prepared);
+  PendingExpectedValue PrepareDelete(int cf, int64_t key) {
+    return latest_->PrepareDelete(cf, key);
   }
 
   // See ExpectedState::PrepareSingleDelete()
-  PendingExpectedValue PrepareSingleDelete(int cf, int64_t key,
-                                           bool* prepared) {
-    return latest_->PrepareSingleDelete(cf, key, prepared);
+  PendingExpectedValue PrepareSingleDelete(int cf, int64_t key) {
+    return latest_->PrepareSingleDelete(cf, key);
   }
 
   // See ExpectedState::PrepareDeleteRange()
   std::vector<PendingExpectedValue> PrepareDeleteRange(int cf,
                                                        int64_t begin_key,
-                                                       int64_t end_key,
-                                                       bool* prepared) {
-    return latest_->PrepareDeleteRange(cf, begin_key, end_key, prepared);
+                                                       int64_t end_key) {
+    return latest_->PrepareDeleteRange(cf, begin_key, end_key);
   }
 
   // See ExpectedState::Exists()

db_stress_tool/expected_value.cc

@@ -10,15 +10,14 @@
 #include <atomic>
 
 namespace ROCKSDB_NAMESPACE {
-bool ExpectedValue::Put(bool pending) {
+void ExpectedValue::Put(bool pending) {
   if (pending) {
     SetPendingWrite();
   } else {
     SetValueBase(NextValueBase());
     ClearDeleted();
     ClearPendingWrite();
   }
-  return true;
 }
 
 bool ExpectedValue::Delete(bool pending) {

db_stress_tool/expected_value.h

@@ -37,11 +37,14 @@ class ExpectedValue {
   explicit ExpectedValue(uint32_t expected_value)
       : expected_value_(expected_value) {}
 
-  bool Exists() const { return !IsDeleted(); }
+  bool Exists() const {
+    assert(!PendingWrite() && !PendingDelete());
+    return !IsDeleted();
+  }
 
   uint32_t Read() const { return expected_value_; }
 
-  bool Put(bool pending);
+  void Put(bool pending);
 
   bool Delete(bool pending);