From 714c63c5840c8ec85b226db5750bd6723aa38561 Mon Sep 17 00:00:00 2001 From: Lei Jin Date: Mon, 27 Oct 2014 12:11:16 -0700 Subject: [PATCH] db_stress for dynamic options Summary: Allow SetOptions() during db_stress test Test Plan: make crash_test Reviewers: sdong, yhchiang, rven, igor Reviewed By: igor Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D25497 --- tools/db_crashtest.py | 1 + tools/db_stress.cc | 161 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 8d0b4f5f7..77bd6ef27 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -98,6 +98,7 @@ def main(argv): --filter_deletes=%s --memtablerep=prefix_hash --prefix_size=7 + --set_options_one_in=10000 """ % (ops_per_thread, threads, write_buf_size, diff --git a/tools/db_stress.cc b/tools/db_stress.cc index b5c79bf3b..2cdf241bb 100644 --- a/tools/db_stress.cc +++ b/tools/db_stress.cc @@ -191,6 +191,9 @@ DEFINE_int32(clear_column_family_one_in, 1000000, "it again. If N == 0, never drop/create column families. " "When test_batches_snapshots is true, this flag has no effect"); +DEFINE_int32(set_options_one_in, 0, + "With a chance of 1/N, change some random options"); + DEFINE_int64(cache_size, 2 * KB * KB * KB, "Number of bytes to use as a cache of uncompressed data."); @@ -372,7 +375,7 @@ static bool ValidatePrefixSize(const char* flagname, int32_t value) { return true; } DEFINE_int32(prefix_size, 7, "Control the prefix size for HashSkipListRep"); -static const bool FLAGS_prefix_size_dummy = +static const bool FLAGS_prefix_size_dummy __attribute__((unused)) = RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize); DEFINE_bool(use_merge, false, "On true, replaces all writes with a Merge " @@ -787,8 +790,129 @@ class StressTest { delete db_; } + bool BuildOptionsTable() { + if (FLAGS_set_options_one_in <= 0) { + return true; + } + options_table_ = { + {"write_buffer_size", + { + std::to_string(FLAGS_write_buffer_size), + std::to_string(FLAGS_write_buffer_size * 2), + std::to_string(FLAGS_write_buffer_size * 4) + } + }, + {"max_write_buffer_number", + { + std::to_string(FLAGS_max_write_buffer_number), + std::to_string(FLAGS_max_write_buffer_number * 2), + std::to_string(FLAGS_max_write_buffer_number * 4) + } + }, + {"arena_block_size", + { + std::to_string(Options().arena_block_size), + std::to_string(FLAGS_write_buffer_size / 4), + std::to_string(FLAGS_write_buffer_size / 8), + } + }, + {"memtable_prefix_bloom_bits", {"0", "8", "10"}}, + {"memtable_prefix_bloom_probes", {"4", "5", "6"}}, + {"memtable_prefix_bloom_huge_page_tlb_size", + { + "0", + std::to_string(2 * 1024 * 1024) + } + }, + {"max_successive_merges", {"0", "2", "4"}}, + {"filter_deletes", {"0", "1"}}, + {"inplace_update_num_locks", {"100", "200", "300"}}, + // TODO(ljin): enable test for this option + // {"disable_auto_compactions", {"100", "200", "300"}}, + {"soft_rate_limit", {"0", "0.5", "0.9"}}, + {"hard_rate_limit", {"0", "1.1", "2.0"}}, + {"level0_file_num_compaction_trigger", + { + std::to_string(FLAGS_level0_file_num_compaction_trigger), + std::to_string(FLAGS_level0_file_num_compaction_trigger + 2), + std::to_string(FLAGS_level0_file_num_compaction_trigger + 4), + } + }, + {"level0_slowdown_writes_trigger", + { + std::to_string(FLAGS_level0_slowdown_writes_trigger), + std::to_string(FLAGS_level0_slowdown_writes_trigger + 2), + std::to_string(FLAGS_level0_slowdown_writes_trigger + 4), + } + }, + {"level0_stop_writes_trigger", + { + std::to_string(FLAGS_level0_stop_writes_trigger), + std::to_string(FLAGS_level0_stop_writes_trigger + 2), + std::to_string(FLAGS_level0_stop_writes_trigger + 4), + } + }, + {"max_grandparent_overlap_factor", + { + std::to_string(Options().max_grandparent_overlap_factor - 5), + std::to_string(Options().max_grandparent_overlap_factor), + std::to_string(Options().max_grandparent_overlap_factor + 5), + } + }, + {"expanded_compaction_factor", + { + std::to_string(Options().expanded_compaction_factor - 5), + std::to_string(Options().expanded_compaction_factor), + std::to_string(Options().expanded_compaction_factor + 5), + } + }, + {"source_compaction_factor", + { + std::to_string(Options().source_compaction_factor), + std::to_string(Options().source_compaction_factor * 2), + std::to_string(Options().source_compaction_factor * 4), + } + }, + {"target_file_size_base", + { + std::to_string(FLAGS_target_file_size_base), + std::to_string(FLAGS_target_file_size_base * 2), + std::to_string(FLAGS_target_file_size_base * 4), + } + }, + {"target_file_size_multiplier", + { + std::to_string(FLAGS_target_file_size_multiplier), + "1", + "2", + } + }, + {"max_bytes_for_level_base", + { + std::to_string(FLAGS_max_bytes_for_level_base / 2), + std::to_string(FLAGS_max_bytes_for_level_base), + std::to_string(FLAGS_max_bytes_for_level_base * 2), + } + }, + {"max_bytes_for_level_multiplier", + { + std::to_string(FLAGS_max_bytes_for_level_multiplier), + "1", + "2", + } + }, + {"max_mem_compaction_level", {"0", "1", "2"}}, + {"max_sequential_skip_in_iterations", {"4", "8", "12"}}, + }; + for (const auto& iter : options_table_) { + options_index_.push_back(iter.first); + } + return true; + } + bool Run() { PrintEnv(); + BuildOptionsTable(); Open(); SharedState shared(this); uint32_t n = shared.GetNumThreads(); @@ -1169,6 +1293,33 @@ class StressTest { return s; } + bool SetOptions(ThreadState* thread) { + assert(FLAGS_set_options_one_in > 0); + std::unordered_map opts; + std::string name = options_index_[ + thread->rand.Next() % options_index_.size()]; + int value_idx = thread->rand.Next() % options_table_[name].size(); + if (name == "soft_rate_limit" || name == "hard_rate_limit") { + opts["soft_rate_limit"] = options_table_["soft_rate_limit"][value_idx]; + opts["hard_rate_limit"] = options_table_["hard_rate_limit"][value_idx]; + } else if (name == "level0_file_num_compaction_trigger" || + name == "level0_slowdown_writes_trigger" || + name == "level0_stop_writes_trigger") { + opts["level0_file_num_compaction_trigger"] = + options_table_["level0_file_num_compaction_trigger"][value_idx]; + opts["level0_slowdown_writes_trigger"] = + options_table_["level0_slowdown_writes_trigger"][value_idx]; + opts["level0_stop_writes_trigger"] = + options_table_["level0_stop_writes_trigger"][value_idx]; + } else { + opts[name] = options_table_[name][value_idx]; + } + + int rand_cf_idx = thread->rand.Next() % FLAGS_column_families; + auto cfh = column_families_[rand_cf_idx]; + return db_->SetOptions(cfh, opts); + } + void OperateDb(ThreadState* thread) { ReadOptions read_opts(FLAGS_verify_checksum, true); WriteOptions write_opts; @@ -1205,6 +1356,12 @@ class StressTest { } } + // Change Options + if (FLAGS_set_options_one_in > 0 && + thread->rand.OneIn(FLAGS_set_options_one_in)) { + SetOptions(thread); + } + if (!FLAGS_test_batches_snapshots && FLAGS_clear_column_family_one_in != 0 && FLAGS_column_families > 1) { if (thread->rand.OneIn(FLAGS_clear_column_family_one_in)) { @@ -1751,6 +1908,8 @@ class StressTest { std::vector column_family_names_; std::atomic new_column_family_name_; int num_times_reopened_; + std::unordered_map> options_table_; + std::vector options_index_; }; } // namespace rocksdb