Add file checksum to stress/crash test (#7343)

Summary:
This change has the crash test randomly select from a few file
checksum implementations, or nullptr, for DB file_checksum_gen_factory.
For compatibility across runs on same DB, each non-null factory can
understand all the other functions, but the default changes.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7343

Test Plan:
'make blackbox_crash_test' for a while, including with some
debug output to ensure code is being exercised.

Reviewed By: zhichao-cao

Differential Revision: D23494580

Pulled By: pdillinger

fbshipit-source-id: 73bbc7ca32c1adaf619134c0c830f12894880b8a

db_stress_tool/db_stress_common.cc

@@ -10,8 +10,12 @@
 
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
+
 #include <cmath>
 
+#include "util/file_checksum_helper.h"
+#include "util/xxhash.h"
+
 ROCKSDB_NAMESPACE::DbStressEnvWrapper* db_stress_env = nullptr;
 #ifndef NDEBUG
 // If non-null, injects read error at a rate specified by the
@@ -226,5 +230,106 @@ size_t GenerateValue(uint32_t rand, char* v, size_t max_sz) {
   v[value_sz] = '\0';
   return value_sz;  // the size of the value set.
 }
+
+namespace {
+
+class MyXXH64Checksum : public FileChecksumGenerator {
+ public:
+  explicit MyXXH64Checksum(bool big) : big_(big) {
+    state_ = XXH64_createState();
+    XXH64_reset(state_, 0);
+  }
+
+  virtual ~MyXXH64Checksum() override { XXH64_freeState(state_); }
+
+  void Update(const char* data, size_t n) override {
+    XXH64_update(state_, data, n);
+  }
+
+  void Finalize() override {
+    assert(str_.empty());
+    uint64_t digest = XXH64_digest(state_);
+    // Store as little endian raw bytes
+    PutFixed64(&str_, digest);
+    if (big_) {
+      // Throw in some more data for stress testing (448 bits total)
+      PutFixed64(&str_, GetSliceHash64(str_));
+      PutFixed64(&str_, GetSliceHash64(str_));
+      PutFixed64(&str_, GetSliceHash64(str_));
+      PutFixed64(&str_, GetSliceHash64(str_));
+      PutFixed64(&str_, GetSliceHash64(str_));
+      PutFixed64(&str_, GetSliceHash64(str_));
+    }
+  }
+
+  std::string GetChecksum() const override {
+    assert(!str_.empty());
+    return str_;
+  }
+
+  const char* Name() const override {
+    return big_ ? "MyBigChecksum" : "MyXXH64Checksum";
+  }
+
+ private:
+  bool big_;
+  XXH64_state_t* state_;
+  std::string str_;
+};
+
+class DbStressChecksumGenFactory : public FileChecksumGenFactory {
+  std::string default_func_name_;
+
+  std::unique_ptr<FileChecksumGenerator> CreateFromFuncName(
+      const std::string& func_name) {
+    std::unique_ptr<FileChecksumGenerator> rv;
+    if (func_name == "FileChecksumCrc32c") {
+      rv.reset(new FileChecksumGenCrc32c(FileChecksumGenContext()));
+    } else if (func_name == "MyXXH64Checksum") {
+      rv.reset(new MyXXH64Checksum(false /* big */));
+    } else if (func_name == "MyBigChecksum") {
+      rv.reset(new MyXXH64Checksum(true /* big */));
+    } else {
+      // Should be a recognized function when we get here
+      assert(false);
+    }
+    return rv;
+  }
+
+ public:
+  explicit DbStressChecksumGenFactory(const std::string& default_func_name)
+      : default_func_name_(default_func_name) {}
+
+  std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
+      const FileChecksumGenContext& context) override {
+    if (context.requested_checksum_func_name.empty()) {
+      return CreateFromFuncName(default_func_name_);
+    } else {
+      return CreateFromFuncName(context.requested_checksum_func_name);
+    }
+  }
+
+  const char* Name() const override { return "FileChecksumGenCrc32cFactory"; }
+};
+
+}  // namespace
+
+std::shared_ptr<FileChecksumGenFactory> GetFileChecksumImpl(
+    const std::string& name) {
+  // Translate from friendly names to internal names
+  std::string internal_name;
+  if (name == "crc32c") {
+    internal_name = "FileChecksumCrc32c";
+  } else if (name == "xxh64") {
+    internal_name = "MyXXH64Checksum";
+  } else if (name == "big") {
+    internal_name = "MyBigChecksum";
+  } else {
+    assert(name.empty() || name == "none");
+    return nullptr;
+  }
+  return std::make_shared<DbStressChecksumGenFactory>(internal_name);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 #endif  // GFLAGS

db_stress_tool/db_stress_common.h

@@ -225,6 +225,7 @@ DECLARE_int32(verify_checksum_one_in);
 DECLARE_int32(verify_db_one_in);
 DECLARE_int32(continuous_verification_interval);
 DECLARE_int32(get_property_one_in);
+DECLARE_string(file_checksum_impl);
 
 #ifndef ROCKSDB_LITE
 DECLARE_bool(use_blob_db);
@@ -541,5 +542,8 @@ extern StressTest* CreateBatchedOpsStressTest();
 extern StressTest* CreateNonBatchedOpsStressTest();
 extern void InitializeHotKeyGenerator(double alpha);
 extern int64_t GetOneHotKeyID(double rand_seed, int64_t max_key);
+
+std::shared_ptr<FileChecksumGenFactory> GetFileChecksumImpl(
+    const std::string& name);
 }  // namespace ROCKSDB_NAMESPACE
 #endif  // GFLAGS

db_stress_tool/db_stress_gflags.cc

@@ -728,4 +728,8 @@ DEFINE_bool(enable_compaction_filter, false,
             "If true, configures a compaction filter that returns a kRemove "
             "decision for deleted keys.");
 
+DEFINE_string(file_checksum_impl, "none",
+              "Name of an implementation for file_checksum_gen_factory, or "
+              "\"none\" for null.");
+
 #endif  // GFLAGS

db_stress_tool/db_stress_test_base.cc

@@ -1838,6 +1838,8 @@ void StressTest::PrintEnv() const {
           bottommost_compression.c_str());
   std::string checksum = ChecksumTypeToString(checksum_type_e);
   fprintf(stdout, "Checksum type             : %s\n", checksum.c_str());
+  fprintf(stdout, "File checksum impl        : %s\n",
+          FLAGS_file_checksum_impl.c_str());
   fprintf(stdout, "Bloom bits / key          : %s\n",
           FormatDoubleParam(FLAGS_bloom_bits).c_str());
   fprintf(stdout, "Max subcompactions        : %" PRIu64 "\n",
@@ -1989,6 +1991,8 @@ void StressTest::Open() {
         FLAGS_max_write_batch_group_size_bytes;
     options_.level_compaction_dynamic_level_bytes =
         FLAGS_level_compaction_dynamic_level_bytes;
+    options_.file_checksum_gen_factory =
+        GetFileChecksumImpl(FLAGS_file_checksum_impl);
   } else {
 #ifdef ROCKSDB_LITE
     fprintf(stderr, "--options_file not supported in lite mode\n");

tools/db_crashtest.py

@@ -61,6 +61,7 @@ default_params = {
     "enable_compaction_filter": lambda: random.choice([0, 0, 0, 1]),
     "expected_values_path": expected_values_file.name,
     "flush_one_in": 1000000,
+    "file_checksum_impl": lambda: random.choice(["none", "crc32c", "xxh64", "big"]),
     "get_live_files_one_in": 1000000,
     # Note: the following two are intentionally disabled as the corresponding
     # APIs are not guaranteed to succeed.