From 88acd932f63b89c13aa1a7faccd937f9c8875527 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Thu, 2 Jun 2016 16:24:14 -0700 Subject: [PATCH] Allows db_bench to take an options file Summary: This patch allows db_bench to initialize it's RocksDB Options via a options file, specified by the --options_file flag. Note that if --options_file flag is set, then it has higher priority than the command-line argument. Test Plan: db_bench_tool_test Reviewers: sdong, IslamAbdelRahman, kradhakrishnan, yiwu, andrewkr Reviewed By: andrewkr Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D58533 --- Makefile | 3 + src.mk | 1 + tools/db_bench_tool.cc | 138 ++++++++++----- tools/db_bench_tool_test.cc | 323 ++++++++++++++++++++++++++++++++++++ 4 files changed, 425 insertions(+), 40 deletions(-) create mode 100644 tools/db_bench_tool_test.cc diff --git a/Makefile b/Makefile index b2749957d..c1c003303 100644 --- a/Makefile +++ b/Makefile @@ -1117,6 +1117,9 @@ options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS) options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +db_bench_tool_test: tools/db_bench_tool_test.o $(BENCHTOOLOBJECTS) $(TESTHARNESS) + $(AM_LINK) + event_logger_test: util/event_logger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) diff --git a/src.mk b/src.mk index eca05222b..16b1001bd 100644 --- a/src.mk +++ b/src.mk @@ -249,6 +249,7 @@ TEST_BENCH_SOURCES = \ table/merger_test.cc \ table/table_reader_bench.cc \ table/table_test.cc \ + tools/db_bench_tool_test.cc \ tools/db_sanity_test.cc \ tools/ldb_cmd_test.cc \ tools/reduce_levels_test.cc \ diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 5f59791f0..f40ed8ad9 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -51,6 +51,7 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/utilities/flashcache.h" #include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/options_util.h" #include "rocksdb/utilities/sim_cache.h" #include "rocksdb/utilities/transaction.h" #include "rocksdb/utilities/transaction_db.h" @@ -530,6 +531,23 @@ DEFINE_int32(transaction_sleep, 0, DEFINE_uint64(transaction_lock_timeout, 100, "If using a transaction_db, specifies the lock wait timeout in" " milliseconds before failing a transaction waiting on a lock"); +DEFINE_string( + options_file, "", + "The path to a RocksDB options file. If specified, then db_bench will " + "run with the RocksDB options in the default column family of the " + "specified options file. " + "Note that with this setting, db_bench will ONLY accept the following " + "RocksDB options related command-line arguments, all other arguments " + "that are related to RocksDB options will be ignored:\n" + "\t--use_existing_db\n" + "\t--statistics\n" + "\t--row_cache_size\n" + "\t--row_cache_numshardbits\n" + "\t--enable_io_prio\n" + "\t--disable_flashcache_for_background_threads\n" + "\t--flashcache_dev\n" + "\t--dump_malloc_stats\n" + "\t--num_multi_db\n"); #endif // ROCKSDB_LITE DEFINE_bool(report_bg_io_stats, false, @@ -2385,13 +2403,36 @@ class Benchmark { } } - void Open(Options* opts) { + // Returns true if the options is initialized from the specified + // options file. + bool InitializeOptionsFromFile(Options* opts) { +#ifndef ROCKSDB_LITE + printf("Initializing RocksDB Options from the specified file\n"); + DBOptions db_opts; + std::vector cf_descs; + if (FLAGS_options_file != "") { + auto s = LoadOptionsFromFile(FLAGS_options_file, Env::Default(), &db_opts, + &cf_descs); + if (s.ok()) { + *opts = Options(db_opts, cf_descs[0].options); + return true; + } + fprintf(stderr, "Unable to load options file %s --- %s\n", + FLAGS_options_file.c_str(), s.ToString().c_str()); + exit(1); + } +#endif + return false; + } + + void InitializeOptionsFromFlags(Options* opts) { + printf("Initializing RocksDB Options from command-line flags\n"); Options& options = *opts; assert(db_.db == nullptr); - options.create_if_missing = !FLAGS_use_existing_db; options.create_missing_column_families = FLAGS_num_column_families > 1; + options.max_open_files = FLAGS_open_files; options.db_write_buffer_size = FLAGS_db_write_buffer_size; options.write_buffer_size = FLAGS_write_buffer_size; options.max_write_buffer_number = FLAGS_max_write_buffer_number; @@ -2417,39 +2458,14 @@ class Benchmark { } options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits; options.bloom_locality = FLAGS_bloom_locality; - options.max_open_files = FLAGS_open_files; options.max_file_opening_threads = FLAGS_file_opening_threads; options.new_table_reader_for_compaction_inputs = FLAGS_new_table_reader_for_compaction_inputs; options.compaction_readahead_size = FLAGS_compaction_readahead_size; options.random_access_max_buffer_size = FLAGS_random_access_max_buffer_size; options.writable_file_max_buffer_size = FLAGS_writable_file_max_buffer_size; - options.statistics = dbstats; - if (FLAGS_enable_io_prio) { - FLAGS_env->LowerThreadPoolIOPriority(Env::LOW); - FLAGS_env->LowerThreadPoolIOPriority(Env::HIGH); - } - if (FLAGS_disable_flashcache_for_background_threads && - cachedev_fd_ == -1) { - // Avoid creating the env twice when an use_existing_db is true - cachedev_fd_ = open(FLAGS_flashcache_dev.c_str(), O_RDONLY); - if (cachedev_fd_ < 0) { - fprintf(stderr, "Open flash device failed\n"); - exit(1); - } - flashcache_aware_env_ = NewFlashcacheAwareEnv(FLAGS_env, cachedev_fd_); - if (flashcache_aware_env_.get() == nullptr) { - fprintf(stderr, "Failed to open flashcache device at %s\n", - FLAGS_flashcache_dev.c_str()); - std::abort(); - } - options.env = flashcache_aware_env_.get(); - } else { - options.env = FLAGS_env; - } options.disableDataSync = FLAGS_disable_data_sync; options.use_fsync = FLAGS_use_fsync; - options.wal_dir = FLAGS_wal_dir; options.num_levels = FLAGS_num_levels; options.target_file_size_base = FLAGS_target_file_size_base; options.target_file_size_multiplier = FLAGS_target_file_size_multiplier; @@ -2459,14 +2475,6 @@ class Benchmark { options.max_bytes_for_level_multiplier = FLAGS_max_bytes_for_level_multiplier; options.filter_deletes = FLAGS_filter_deletes; - if (FLAGS_row_cache_size) { - if (FLAGS_cache_numshardbits >= 1) { - options.row_cache = - NewLRUCache(FLAGS_row_cache_size, FLAGS_cache_numshardbits); - } else { - options.row_cache = NewLRUCache(FLAGS_row_cache_size); - } - } if ((FLAGS_prefix_size == 0) && (FLAGS_rep_factory == kPrefixHash || FLAGS_rep_factory == kHashLinkedList)) { fprintf(stderr, "prefix_size should be non-zero if PrefixHash or " @@ -2689,6 +2697,48 @@ class Benchmark { } #endif // ROCKSDB_LITE + if (FLAGS_min_level_to_compress >= 0) { + options.compression_per_level.clear(); + } + } + + void InitializeOptionsGeneral(Options* opts) { + Options& options = *opts; + + options.statistics = dbstats; + options.wal_dir = FLAGS_wal_dir; + options.create_if_missing = !FLAGS_use_existing_db; + + if (FLAGS_row_cache_size) { + if (FLAGS_cache_numshardbits >= 1) { + options.row_cache = + NewLRUCache(FLAGS_row_cache_size, FLAGS_cache_numshardbits); + } else { + options.row_cache = NewLRUCache(FLAGS_row_cache_size); + } + } + if (FLAGS_enable_io_prio) { + FLAGS_env->LowerThreadPoolIOPriority(Env::LOW); + FLAGS_env->LowerThreadPoolIOPriority(Env::HIGH); + } + if (FLAGS_disable_flashcache_for_background_threads && cachedev_fd_ == -1) { + // Avoid creating the env twice when an use_existing_db is true + cachedev_fd_ = open(FLAGS_flashcache_dev.c_str(), O_RDONLY); + if (cachedev_fd_ < 0) { + fprintf(stderr, "Open flash device failed\n"); + exit(1); + } + flashcache_aware_env_ = NewFlashcacheAwareEnv(FLAGS_env, cachedev_fd_); + if (flashcache_aware_env_.get() == nullptr) { + fprintf(stderr, "Failed to open flashcache device at %s\n", + FLAGS_flashcache_dev.c_str()); + std::abort(); + } + options.env = flashcache_aware_env_.get(); + } else { + options.env = FLAGS_env; + } + if (FLAGS_num_multi_db <= 1) { OpenDb(options, FLAGS_db, &db_); } else { @@ -2698,11 +2748,15 @@ class Benchmark { OpenDb(options, GetDbNameForMultiple(FLAGS_db, i), &multi_dbs_[i]); } } - if (FLAGS_min_level_to_compress >= 0) { - options.compression_per_level.clear(); + options.dump_malloc_stats = FLAGS_dump_malloc_stats; + } + + void Open(Options* opts) { + if (!InitializeOptionsFromFile(opts)) { + InitializeOptionsFromFlags(opts); } - options.dump_malloc_stats = FLAGS_dump_malloc_stats; + InitializeOptionsGeneral(opts); } void OpenDb(const Options& options, const std::string& db_name, @@ -3992,8 +4046,12 @@ class Benchmark { int db_bench_tool(int argc, char** argv) { rocksdb::port::InstallStackTraceHandler(); - SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + - " [OPTIONS]..."); + static bool initialized = false; + if (!initialized) { + SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + + " [OPTIONS]..."); + initialized = true; + } ParseCommandLineFlags(&argc, &argv, true); FLAGS_compaction_style_e = (rocksdb::CompactionStyle) FLAGS_compaction_style; diff --git a/tools/db_bench_tool_test.cc b/tools/db_bench_tool_test.cc new file mode 100644 index 000000000..4088483f4 --- /dev/null +++ b/tools/db_bench_tool_test.cc @@ -0,0 +1,323 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/db_bench_tool.h" +#include "rocksdb/utilities/options_util.h" +#include "util/options_parser.h" +#include "util/random.h" +#include "util/testharness.h" +#include "util/testutil.h" + +#ifdef GFLAGS +#include + +namespace rocksdb { +namespace { +static const int kMaxArgCount = 100; +static const size_t kArgBufferSize = 100000; +} // namespace + +class DBBenchTest : public testing::Test { + public: + DBBenchTest() : rnd_(0xFB) { + test_path_ = test::TmpDir() + "/db_bench_test"; + Env::Default()->CreateDir(test_path_); + db_path_ = test_path_ + "/db"; + wal_path_ = test_path_ + "/wal"; + } + + ~DBBenchTest() { + // DestroyDB(db_path_, Options()); + } + + void ResetArgs() { + argc_ = 0; + cursor_ = 0; + memset(arg_buffer_, 0, kArgBufferSize); + } + + void AppendArgs(const std::vector& args) { + for (const auto& arg : args) { + ASSERT_LE(cursor_ + arg.size() + 1, kArgBufferSize); + ASSERT_LE(argc_ + 1, kMaxArgCount); + snprintf(arg_buffer_ + cursor_, arg.size() + 1, "%s", arg.c_str()); + + argv_[argc_++] = arg_buffer_ + cursor_; + cursor_ += arg.size() + 1; + } + } + + void RunDbBench(const std::string& options_file_name) { + AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0", + "--num=1000", + std::string(std::string("--db=") + db_path_).c_str(), + std::string(std::string("--wal_dir=") + wal_path_).c_str(), + std::string(std::string("--options_file=") + options_file_name) + .c_str()}); + ASSERT_EQ(0, db_bench_tool(argc(), argv())); + } + + void VerifyOptions(const Options& opt) { + DBOptions loaded_db_opts; + std::vector cf_descs; + ASSERT_OK(LoadLatestOptions(db_path_, Env::Default(), &loaded_db_opts, + &cf_descs)); + + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(DBOptions(opt), loaded_db_opts)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(opt), + cf_descs[0].options)); + + // check with the default rocksdb options and expect failure + ASSERT_NOK( + RocksDBOptionsParser::VerifyDBOptions(DBOptions(), loaded_db_opts)); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(), + cf_descs[0].options)); + } + + char** argv() { return argv_; } + + int argc() { return argc_; } + + std::string db_path_; + std::string test_path_; + std::string wal_path_; + + char arg_buffer_[kArgBufferSize]; + char* argv_[kMaxArgCount]; + int argc_ = 0; + int cursor_ = 0; + Random rnd_; +}; + +namespace {} // namespace + +TEST_F(DBBenchTest, OptionsFile) { + const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; + + Options opt; + opt.create_if_missing = true; + opt.max_open_files = 256; + opt.base_background_compactions = 5; + opt.max_background_compactions = 10; + opt.arena_block_size = 8388608; + ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, + {ColumnFamilyOptions(opt)}, kOptionsFileName, + Env::Default())); + + // override the following options as db_bench will not take these + // options from the options file + opt.wal_dir = wal_path_; + + RunDbBench(kOptionsFileName); + + VerifyOptions(opt); +} + +TEST_F(DBBenchTest, OptionsFileUniversal) { + const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; + + Options opt; + opt.compaction_style = kCompactionStyleUniversal; + opt.num_levels = 1; + opt.create_if_missing = true; + opt.max_open_files = 256; + opt.base_background_compactions = 5; + opt.max_background_compactions = 10; + opt.arena_block_size = 8388608; + ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, + {ColumnFamilyOptions(opt)}, kOptionsFileName, + Env::Default())); + + // override the following options as db_bench will not take these + // options from the options file + opt.wal_dir = wal_path_; + + RunDbBench(kOptionsFileName); + + VerifyOptions(opt); +} + +TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) { + const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; + + Options opt; + opt.compaction_style = kCompactionStyleUniversal; + opt.num_levels = 12; + opt.create_if_missing = true; + opt.max_open_files = 256; + opt.base_background_compactions = 5; + opt.max_background_compactions = 10; + opt.arena_block_size = 8388608; + ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, + {ColumnFamilyOptions(opt)}, kOptionsFileName, + Env::Default())); + + // override the following options as db_bench will not take these + // options from the options file + opt.wal_dir = wal_path_; + + RunDbBench(kOptionsFileName); + + VerifyOptions(opt); +} + +const std::string options_file_content = R"OPTIONS_FILE( +[Version] + rocksdb_version=4.3.1 + options_file_version=1.1 + +[DBOptions] + wal_bytes_per_sync=1048576 + delete_obsolete_files_period_micros=0 + WAL_ttl_seconds=0 + WAL_size_limit_MB=0 + db_write_buffer_size=0 + max_subcompactions=1 + table_cache_numshardbits=4 + max_open_files=-1 + max_file_opening_threads=10 + base_background_compactions=3 + max_background_compactions=5 + use_fsync=false + use_adaptive_mutex=false + max_total_wal_size=18446744073709551615 + compaction_readahead_size=0 + new_table_reader_for_compaction_inputs=false + keep_log_file_num=10 + skip_stats_update_on_db_open=false + max_manifest_file_size=18446744073709551615 + db_log_dir= + skip_log_error_on_recovery=false + writable_file_max_buffer_size=1048576 + paranoid_checks=true + is_fd_close_on_exec=true + bytes_per_sync=1048576 + enable_thread_tracking=true + disable_data_sync=false + recycle_log_file_num=0 + disableDataSync=false + create_missing_column_families=false + log_file_time_to_roll=0 + max_background_flushes=1 + create_if_missing=true + error_if_exists=false + allow_os_buffer=true + delayed_write_rate=1048576 + manifest_preallocation_size=4194304 + allow_mmap_writes=false + stats_dump_period_sec=600 + allow_fallocate=true + allow_mmap_reads=false + max_log_file_size=83886080 + random_access_max_buffer_size=1048576 + advise_random_on_open=true + + +[CFOptions "default"] + compaction_filter_factory=nullptr + table_factory=BlockBasedTable + prefix_extractor=nullptr + comparator=leveldb.BytewiseComparator + compression_per_level= + max_bytes_for_level_base=104857600 + bloom_locality=0 + target_file_size_base=10485760 + memtable_prefix_bloom_huge_page_tlb_size=0 + max_successive_merges=1000 + max_sequential_skip_in_iterations=8 + arena_block_size=52428800 + target_file_size_multiplier=1 + source_compaction_factor=1 + min_write_buffer_number_to_merge=1 + max_write_buffer_number=2 + write_buffer_size=419430400 + max_grandparent_overlap_factor=10 + max_bytes_for_level_multiplier=10 + memtable_factory=SkipListFactory + compression=kSnappyCompression + min_partial_merge_operands=2 + level0_stop_writes_trigger=100 + num_levels=1 + level0_slowdown_writes_trigger=50 + level0_file_num_compaction_trigger=10 + expanded_compaction_factor=25 + soft_rate_limit=0.000000 + max_write_buffer_number_to_maintain=0 + verify_checksums_in_compaction=true + merge_operator=nullptr + memtable_prefix_bloom_bits=0 + paranoid_file_checks=false + inplace_update_num_locks=10000 + optimize_filters_for_hits=false + level_compaction_dynamic_level_bytes=false + inplace_update_support=false + compaction_style=kCompactionStyleUniversal + memtable_prefix_bloom_probes=6 + purge_redundant_kvs_while_flush=true + filter_deletes=false + hard_pending_compaction_bytes_limit=0 + disable_auto_compactions=false + compaction_measure_io_stats=false + +[TableOptions/BlockBasedTable "default"] + format_version=0 + skip_table_builder_flush=false + cache_index_and_filter_blocks=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory + hash_index_allow_collision=true + index_type=kBinarySearch + whole_key_filtering=true + checksum=kCRC32c + no_block_cache=false + block_size=32768 + block_size_deviation=10 + block_restart_interval=16 + filter_policy=rocksdb.BuiltinBloomFilter +)OPTIONS_FILE"; + +TEST_F(DBBenchTest, OptionsFileFromFile) { + const std::string kOptionsFileName = test_path_ + "/OPTIONS_flash"; + unique_ptr writable; + ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName, &writable, + EnvOptions())); + ASSERT_OK(writable->Append(options_file_content)); + ASSERT_OK(writable->Close()); + + DBOptions db_opt; + std::vector cf_descs; + ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt, + &cf_descs)); + Options opt(db_opt, cf_descs[0].options); + + opt.create_if_missing = true; + + // override the following options as db_bench will not take these + // options from the options file + opt.wal_dir = wal_path_; + + RunDbBench(kOptionsFileName); + + VerifyOptions(opt); +} + +} // namespace rocksdb + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} + +#else + +int main(int argc, char** argv) { + printf("Skip db_bench_tool_test as the required library GFLAG is missing."); +} +#endif // #ifdef GFLAGS