minor improvements to db_stress

Summary:
fix some things that made this command hard to use from CLI:

- use default values for `target_file_size_base` and `max_bytes_for_level_base`. previously we were using small values for these but default value of `write_buffer_size`, which led to enormous number of L1 files.
- failure message for `value_size_mult` too big. previously there was just an assert, so in non-debug mode it'd overrun the value buffer and crash mysteriously.
- only print verification success if there's no failure. before it'd print both in the failure case.
- support `memtable_prefix_bloom_size_ratio`
- support `num_bottom_pri_threads` (universal compaction)
Closes https://github.com/facebook/rocksdb/pull/2741

Differential Revision: D5629495

Pulled By: ajkr

fbshipit-source-id: ddad97d6d4ba0884e7c0f933b0a359712514fc1d
main
Andrew Kryczka 7 years ago committed by Facebook Github Bot
parent af012c0f83
commit 23593171c4
  1. 42
      tools/db_stress.cc

@ -72,6 +72,8 @@ using GFLAGS::RegisterFlagValidator;
using GFLAGS::SetUsageMessage;
static const long KB = 1024;
static const int kRandomValueMaxFactor = 3;
static const int kValueMaxLen = 100;
static bool ValidateUint32Range(const char* flagname, uint64_t value) {
if (value > std::numeric_limits<uint32_t>::max()) {
@ -174,6 +176,11 @@ DEFINE_int32(max_write_buffer_number_to_maintain,
"after they are flushed. If this value is set to -1, "
"'max_write_buffer_number' will be used.");
DEFINE_double(memtable_prefix_bloom_size_ratio,
rocksdb::Options().memtable_prefix_bloom_size_ratio,
"creates prefix blooms for memtables, each with size "
"`write_buffer_size * memtable_prefix_bloom_size_ratio`.");
DEFINE_int32(open_files, rocksdb::Options().max_open_files,
"Maximum number of files to keep open at the same time "
"(use default if == 0)");
@ -205,6 +212,10 @@ DEFINE_int32(max_background_compactions,
"The maximum number of concurrent background compactions "
"that can occur in parallel.");
DEFINE_int32(num_bottom_pri_threads, 0,
"The number of threads in the bottom-priority thread pool (used "
"by universal compaction only).");
DEFINE_int32(compaction_thread_pool_adjust_interval, 0,
"The interval (in milliseconds) to adjust compaction thread pool "
"size. Don't change it periodically if the value is 0.");
@ -317,13 +328,15 @@ extern std::vector<std::string> rocksdb_kill_prefix_blacklist;
DEFINE_bool(disable_wal, false, "If true, do not write WAL for write.");
DEFINE_int32(target_file_size_base, 64 * KB,
DEFINE_int32(target_file_size_base, rocksdb::Options().target_file_size_base,
"Target level-1 file size for compaction");
DEFINE_int32(target_file_size_multiplier, 1,
"A multiplier to compute target level-N file size (N >= 2)");
DEFINE_uint64(max_bytes_for_level_base, 256 * KB, "Max bytes for level-1");
DEFINE_uint64(max_bytes_for_level_base,
rocksdb::Options().max_bytes_for_level_base,
"Max bytes for level-1");
DEFINE_double(max_bytes_for_level_multiplier, 2,
"A multiplier to compute max bytes for level-N (N >= 2)");
@ -1121,8 +1134,6 @@ class StressTest {
ToString(FLAGS_write_buffer_size / 4),
ToString(FLAGS_write_buffer_size / 8),
}},
{"memtable_prefix_bloom_bits", {"0", "8", "10"}},
{"memtable_prefix_bloom_probes", {"4", "5", "6"}},
{"memtable_huge_page_size", {"0", ToString(2 * 1024 * 1024)}},
{"max_successive_merges", {"0", "2", "4"}},
{"inplace_update_num_locks", {"100", "200", "300"}},
@ -1250,7 +1261,7 @@ class StressTest {
threads[i] = nullptr;
}
auto now = FLAGS_env->NowMicros();
if (!FLAGS_test_batches_snapshots) {
if (!FLAGS_test_batches_snapshots && !shared.HasVerificationFailedYet()) {
fprintf(stdout, "%s Verification successful\n",
FLAGS_env->TimeToString(now/1000000).c_str());
}
@ -2021,7 +2032,7 @@ class StressTest {
return false;
}
// compare value_from_db with the value in the shared state
char value[100];
char value[kValueMaxLen];
uint32_t value_base = shared->Get(cf, key);
if (value_base == SharedState::SENTINEL && !strict) {
return true;
@ -2064,7 +2075,8 @@ class StressTest {
}
static size_t GenerateValue(uint32_t rand, char *v, size_t max_sz) {
size_t value_sz = ((rand % 3) + 1) * FLAGS_value_size_mult;
size_t value_sz =
((rand % kRandomValueMaxFactor) + 1) * FLAGS_value_size_mult;
assert(value_sz <= max_sz && value_sz >= sizeof(uint32_t));
*((uint32_t*)v) = rand;
for (size_t i=sizeof(uint32_t); i < value_sz; i++) {
@ -2162,6 +2174,8 @@ class StressTest {
FLAGS_min_write_buffer_number_to_merge;
options_.max_write_buffer_number_to_maintain =
FLAGS_max_write_buffer_number_to_maintain;
options_.memtable_prefix_bloom_size_ratio =
FLAGS_memtable_prefix_bloom_size_ratio;
options_.max_background_compactions = FLAGS_max_background_compactions;
options_.max_background_flushes = FLAGS_max_background_flushes;
options_.compaction_style =
@ -2406,7 +2420,8 @@ int main(int argc, char** argv) {
// The number of background threads should be at least as much the
// max number of concurrent compactions.
FLAGS_env->SetBackgroundThreads(FLAGS_max_background_compactions);
FLAGS_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
rocksdb::Env::Priority::BOTTOM);
if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size <= 0) {
fprintf(stderr,
"Error: prefixpercent is non-zero while prefix_size is "
@ -2419,6 +2434,12 @@ int main(int argc, char** argv) {
"test_batches_snapshots test!\n");
exit(1);
}
if (FLAGS_memtable_prefix_bloom_size_ratio > 0.0 && FLAGS_prefix_size <= 0) {
fprintf(stderr,
"Error: please specify positive prefix_size in order to use "
"memtable_prefix_bloom_size_ratio\n");
exit(1);
}
if ((FLAGS_readpercent + FLAGS_prefixpercent +
FLAGS_writepercent + FLAGS_delpercent + FLAGS_delrangepercent +
FLAGS_iterpercent) != 100) {
@ -2450,6 +2471,11 @@ int main(int argc, char** argv) {
} else if (FLAGS_active_width == 0) {
FLAGS_active_width = FLAGS_max_key;
}
if (FLAGS_value_size_mult * kRandomValueMaxFactor > kValueMaxLen) {
fprintf(stderr, "Error: value_size_mult can be at most %d\n",
kValueMaxLen / kRandomValueMaxFactor);
exit(1);
}
// Choose a location for the test database if none given with --db=<path>
if (FLAGS_db.empty()) {

Loading…
Cancel
Save