From 6380df5e10a1fe91ad0e7cac233b4de5ffc4a383 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Tue, 10 Dec 2019 08:38:23 -0800 Subject: [PATCH] Vary bloom_bits in db_crashtest (#6103) Summary: Especially with non-integral bits/key now supported, db_crashtest should vary the bloom_bits configuration. The probabilities look like this: 1/2 chance of a uniform int from 0 to 19. This includes overall 1/40 chance of 0 which disables the bloom filter. 1/2 chance of a float from a lognormal distribution with a median of 10. This always produces positive values but with a decent chance of < 1 (overall ~1/40) or > 100 (overall ~1/40), the enforced/coerced implementation limits. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6103 Test Plan: start 'make blackbox_crash_test' several times and look at configuration output Differential Revision: D18734877 Pulled By: pdillinger fbshipit-source-id: 4a38cb057d3b3fc1327f93199f65b9a9ffbd7316 --- db_stress_tool/db_stress_common.h | 13 ++++++++++++- db_stress_tool/db_stress_gflags.cc | 6 +++--- db_stress_tool/db_stress_test_base.cc | 2 ++ db_stress_tool/db_stress_tool.cc | 4 ++++ tools/db_crashtest.py | 2 ++ 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 2a1f340a2..fb9b5f9f0 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -133,7 +133,7 @@ DECLARE_uint64(compaction_ttl); DECLARE_bool(allow_concurrent_memtable_write); DECLARE_bool(enable_write_thread_adaptive_yield); DECLARE_int32(reopen); -DECLARE_int32(bloom_bits); +DECLARE_double(bloom_bits); DECLARE_bool(use_block_based_filter); DECLARE_bool(partition_filters); DECLARE_int32(index_type); @@ -342,6 +342,17 @@ extern inline std::string StringToHex(const std::string& str) { return result; } +// Unified output format for double parameters +extern inline std::string FormatDoubleParam(double param) { + return std::to_string(param); +} + +// Make sure that double parameter is a value we can reproduce by +// re-inputting the value printed. +extern inline void SanitizeDoubleParam(double* param) { + *param = std::atof(FormatDoubleParam(*param).c_str()); +} + extern void PoolSizeChangeThread(void* v); extern void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index fbbdc04e1..e1f01b14d 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -275,9 +275,9 @@ DEFINE_int32(reopen, 10, "Number of times database reopens"); static const bool FLAGS_reopen_dummy __attribute__((__unused__)) = RegisterFlagValidator(&FLAGS_reopen, &ValidateInt32Positive); -DEFINE_int32(bloom_bits, 10, - "Bloom filter bits per key. " - "Negative means use default settings."); +DEFINE_double(bloom_bits, 10, + "Bloom filter bits per key. " + "Negative means use default settings."); DEFINE_bool(use_block_based_filter, false, "use block based filter" diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 301d5b8e4..c1d254438 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -1310,6 +1310,8 @@ void StressTest::PrintEnv() const { fprintf(stdout, "Compression : %s\n", compression.c_str()); std::string checksum = ChecksumTypeToString(FLAGS_checksum_type_e); fprintf(stdout, "Checksum type : %s\n", checksum.c_str()); + fprintf(stdout, "Bloom bits / key : %s\n", + FormatDoubleParam(FLAGS_bloom_bits).c_str()); fprintf(stdout, "Max subcompactions : %" PRIu64 "\n", FLAGS_subcompactions); fprintf(stdout, "Use MultiGet : %s\n", diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index 4ce42d0dd..5fb6dee00 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -34,6 +34,10 @@ int db_stress_tool(int argc, char** argv) { " [OPTIONS]..."); ParseCommandLineFlags(&argc, &argv, true); + SanitizeDoubleParam(&FLAGS_bloom_bits); + SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio); + SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier); + if (FLAGS_statistics) { dbstats = rocksdb::CreateDBStatistics(); if (FLAGS_enable_secondary) { diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 1ff7f40c8..9f9c83a6b 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -25,6 +25,8 @@ expected_values_file = tempfile.NamedTemporaryFile() default_params = { "acquire_snapshot_one_in": 10000, "block_size": 16384, + "bloom_bits": lambda: random.choice([random.randint(0,19), + random.lognormvariate(2.3, 1.3)]), "cache_index_and_filter_blocks": lambda: random.randint(0, 1), "cache_size": 1048576, "checkpoint_one_in": 1000000,