Add env_fault_injection argument to db_stress (#6687)

Summary:
Add env_fault_injection argument to db_stress.  When enabled,
FaultInjectionTestEnv will be used instead.  Currently this
option does not support running with other env setting.

This will allow
us to later manually produce error when running db_crashtest.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6687

Test Plan:
make db_stress -j32
./db_stress --env_fault_injection
./db_stress --env_fault_injection --hdfs   // expect error message

Reviewed By: ajkr

Differential Revision: D21014683

Pulled By: yhchiang

fbshipit-source-id: 0724aeac37efd57adb72a37defe6dbd3bfa8106a

db_stress_tool/db_stress_common.h

@@ -70,6 +70,7 @@
 #include "util/string_util.h"
 #include "utilities/blob_db/blob_db.h"
 #include "test_util/testutil.h"
+#include "test_util/fault_injection_test_env.h"
 
 #include "utilities/merge_operators.h"
 
@@ -229,6 +230,7 @@ DECLARE_bool(blob_db_enable_gc);
 DECLARE_double(blob_db_gc_cutoff);
 #endif  // !ROCKSDB_LITE
 DECLARE_int32(approximate_size_one_in);
+DECLARE_bool(sync_fault_injection);
 
 const long KB = 1024;
 const int kRandomValueMaxFactor = 3;

db_stress_tool/db_stress_driver.cc

@@ -61,6 +61,12 @@ bool RunStressTest(StressTest* stress) {
     stress->InitReadonlyDb(&shared);
   }
 
+#ifndef NDEBUG
+  if (FLAGS_sync_fault_injection) {
+    fault_fs_guard->SetFilesystemDirectWritable(false);
+  }
+#endif
+
   uint32_t n = shared.GetNumThreads();
 
   uint64_t now = db_stress_env->NowMicros();

db_stress_tool/db_stress_gflags.cc

@@ -674,4 +674,8 @@ DEFINE_int32(approximate_size_one_in, 64,
 
 DEFINE_int32(read_fault_one_in, 1000,
             "On non-zero, enables fault injection on read");
+
+DEFINE_bool(sync_fault_injection, false,
+            "If true, FaultInjectionTestFS will be used for write operations, "
+            " and unsynced data in DB will lost after crash.");
 #endif  // GFLAGS

db_stress_tool/db_stress_test_base.cc

@@ -1728,6 +1728,7 @@ void StressTest::PrintEnv() const {
   fprintf(stdout, "Use dynamic level         : %d\n",
           static_cast<int>(FLAGS_level_compaction_dynamic_level_bytes));
   fprintf(stdout, "Read fault one in         : %d\n", FLAGS_read_fault_one_in);
+  fprintf(stdout, "Sync fault injection      : %d\n", FLAGS_sync_fault_injection);
 
   fprintf(stdout, "------------------------------------------------\n");
 }

db_stress_tool/db_stress_tool.cc

@@ -73,8 +73,9 @@ int db_stress_tool(int argc, char** argv) {
   } else {
     raw_env = Env::Default();
   }
+
 #ifndef NDEBUG
-  if (FLAGS_read_fault_one_in) {
+  if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection) {
     FaultInjectionTestFS* fs =
         new FaultInjectionTestFS(raw_env->GetFileSystem());
     fault_fs_guard.reset(fs);
@@ -84,6 +85,7 @@ int db_stress_tool(int argc, char** argv) {
     raw_env = fault_env_guard.get();
   }
 #endif
+
   env_wrapper_guard = std::make_shared<DbStressEnvWrapper>(raw_env);
   db_stress_env = env_wrapper_guard.get();
 

test_util/fault_injection_test_fs.cc

@@ -241,15 +241,8 @@ IOStatus FaultInjectionTestFS::NewWritableFile(
   if (IsFilesystemDirectWritable()) {
     return target()->NewWritableFile(fname, file_opts, result, dbg);
   }
-  // Not allow overwriting files
+
-  IOStatus io_s = target()->FileExists(fname, IOOptions(), dbg);
+  IOStatus io_s = target()->NewWritableFile(fname, file_opts, result, dbg);
-  if (io_s.ok()) {
-    return IOStatus::Corruption("File already exists.");
-  } else if (!io_s.IsNotFound()) {
-    assert(io_s.IsIOError());
-    return io_s;
-  }
-  io_s = target()->NewWritableFile(fname, file_opts, result, dbg);
   if (io_s.ok()) {
     result->reset(new TestFSWritableFile(fname, std::move(*result), this));
     // WritableFileWriter* file is opened

tools/db_crashtest.py

@@ -114,7 +114,8 @@ default_params = {
     "continuous_verification_interval" : 0,
     "max_key_len": 3,
     "key_len_percent_dist": "1,30,69",
-    "read_fault_one_in": lambda: random.choice([0, 1000])
+    "read_fault_one_in": lambda: random.choice([0, 1000]),
+    "sync_fault_injection": False
 }
 
 _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'