Stress test to support paranoid_file_checks (#7473)

Summary:
It's important to make sure no false positive is reported when options.paranoid_file_checks is used. Add it to stress test and a place holder in crash test. It is disabled in crash test as there appears to be a bug causing false positive.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7473

Test Plan: Run crash test

Reviewed By: ajkr

Differential Revision: D24026939

fbshipit-source-id: 89102acb45cf041776775ce44a4eef4b0f3a380c

db_stress_tool/db_stress_common.h

@@ -241,6 +241,7 @@ DECLARE_bool(sync_fault_injection);
 DECLARE_bool(best_efforts_recovery);
 DECLARE_bool(skip_verifydb);
 DECLARE_bool(enable_compaction_filter);
+DECLARE_bool(paranoid_file_checks);
 
 const long KB = 1024;
 const int kRandomValueMaxFactor = 3;

db_stress_tool/db_stress_gflags.cc

@@ -728,6 +728,10 @@ DEFINE_bool(enable_compaction_filter, false,
             "If true, configures a compaction filter that returns a kRemove "
             "decision for deleted keys.");
 
+DEFINE_bool(paranoid_file_checks, true,
+            "After writing every SST file, reopen it and read all the keys "
+            "and validate checksums");
+
 DEFINE_string(file_checksum_impl, "none",
               "Name of an implementation for file_checksum_gen_factory, or "
               "\"none\" for null.");

db_stress_tool/db_stress_test_base.cc

@@ -2145,6 +2145,7 @@ void StressTest::Open() {
       std::make_shared<DbStressTablePropertiesCollectorFactory>());
 
   options_.best_efforts_recovery = FLAGS_best_efforts_recovery;
+  options_.paranoid_file_checks = FLAGS_paranoid_file_checks;
 
   fprintf(stdout, "DB path: [%s]\n", FLAGS_db.c_str());
 

tools/db_crashtest.py

@@ -134,6 +134,8 @@ default_params = {
     "read_fault_one_in": lambda: random.choice([0, 1000]),
     "sync_fault_injection": False,
     "get_property_one_in": 1000000,
+    # paranoid_file_checks has a bug so it's not yet passed.
+    "paranoid_file_checks": 0,
 }
 
 _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'
@@ -198,6 +200,8 @@ simple_default_params = {
     "test_batches_snapshots": 0,
     "write_buffer_size": 32 * 1024 * 1024,
     "level_compaction_dynamic_level_bytes": False,
+    # "paranoid_file_checks" has a bug so it's not yet passed.
+    "paranoid_file_checks": 0,
 }
 
 blackbox_simple_default_params = {