Allow db_bench and db_stress to set `allow_data_in_errors` (#10171)

Summary:
There is `Options::allow_data_in_errors` that controls whether RocksDB
is allowed to log data, e.g. key, value, etc in LOG files. It is false
by default. However, in db_bench and db_stress, it is often ok to log
data because there is no concern about privacy.

This PR allows db_stress and db_bench to set this option on the command
line, while it remains false by default. Furthermore, make
crash/recovery test driven by db_crashtest.py to opt-in.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10171

Test Plan: Stress test and db_bench

Reviewed By: hx235

Differential Revision: D37163787

Pulled By: riversand963

fbshipit-source-id: 0242f24d292ba15b6faf8ff903963b85d3e011f8
main
Yanqin Jin 3 years ago committed by Facebook GitHub Bot
parent 19345de60d
commit ce419c0f10
  1. 2
      db_stress_tool/db_stress_common.h
  2. 4
      db_stress_tool/db_stress_gflags.cc
  3. 2
      db_stress_tool/db_stress_test_base.cc
  4. 7
      tools/db_bench_tool.cc
  5. 1
      tools/db_crashtest.py

@ -298,6 +298,8 @@ DECLARE_bool(verify_sst_unique_id_in_manifest);
DECLARE_int32(create_timestamped_snapshot_one_in); DECLARE_int32(create_timestamped_snapshot_one_in);
DECLARE_bool(allow_data_in_errors);
constexpr long KB = 1024; constexpr long KB = 1024;
constexpr int kRandomValueMaxFactor = 3; constexpr int kRandomValueMaxFactor = 3;
constexpr int kValueMaxLen = 100; constexpr int kValueMaxLen = 100;

@ -976,4 +976,8 @@ DEFINE_int32(
create_timestamped_snapshot_one_in, 0, create_timestamped_snapshot_one_in, 0,
"On non-zero, create timestamped snapshots upon transaction commits."); "On non-zero, create timestamped snapshots upon transaction commits.");
DEFINE_bool(allow_data_in_errors,
ROCKSDB_NAMESPACE::Options().allow_data_in_errors,
"If true, allow logging data, e.g. key, value in LOG files.");
#endif // GFLAGS #endif // GFLAGS

@ -2931,6 +2931,8 @@ void InitializeOptionsFromFlags(
if (FLAGS_user_timestamp_size > 0) { if (FLAGS_user_timestamp_size > 0) {
CheckAndSetOptionsForUserTimestamp(options); CheckAndSetOptionsForUserTimestamp(options);
} }
options.allow_data_in_errors = FLAGS_allow_data_in_errors;
} }
void InitializeOptionsGeneral( void InitializeOptionsGeneral(

@ -1646,6 +1646,10 @@ static const bool FLAGS_readwritepercent_dummy __attribute__((__unused__)) =
DEFINE_int32(disable_seek_compaction, false, DEFINE_int32(disable_seek_compaction, false,
"Not used, left here for backwards compatibility"); "Not used, left here for backwards compatibility");
DEFINE_bool(allow_data_in_errors,
ROCKSDB_NAMESPACE::Options().allow_data_in_errors,
"If true, allow logging data, e.g. key, value in LOG files.");
static const bool FLAGS_deletepercent_dummy __attribute__((__unused__)) = static const bool FLAGS_deletepercent_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_deletepercent, &ValidateInt32Percent); RegisterFlagValidator(&FLAGS_deletepercent, &ValidateInt32Percent);
static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((__unused__)) = static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((__unused__)) =
@ -4450,6 +4454,8 @@ class Benchmark {
options.comparator = test::BytewiseComparatorWithU64TsWrapper(); options.comparator = test::BytewiseComparatorWithU64TsWrapper();
} }
options.allow_data_in_errors = FLAGS_allow_data_in_errors;
// Integrated BlobDB // Integrated BlobDB
options.enable_blob_files = FLAGS_enable_blob_files; options.enable_blob_files = FLAGS_enable_blob_files;
options.min_blob_size = FLAGS_min_blob_size; options.min_blob_size = FLAGS_min_blob_size;
@ -4477,7 +4483,6 @@ class Benchmark {
exit(1); exit(1);
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} }
void InitializeOptionsGeneral(Options* opts) { void InitializeOptionsGeneral(Options* opts) {

@ -178,6 +178,7 @@ default_params = {
"wal_compression": lambda: random.choice(["none", "zstd"]), "wal_compression": lambda: random.choice(["none", "zstd"]),
"verify_sst_unique_id_in_manifest": 1, # always do unique_id verification "verify_sst_unique_id_in_manifest": 1, # always do unique_id verification
"secondary_cache_uri": "", "secondary_cache_uri": "",
"allow_data_in_errors": True,
} }
_TEST_DIR_ENV_VAR = 'TEST_TMPDIR' _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'

Loading…
Cancel
Save