diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 36c1108b1..2c9734d24 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "rocksdb/version.h" #include "rocksdb/universal_compaction.h" @@ -1012,6 +1013,12 @@ extern Options GetOptions(size_t total_write_buffer_limit, int read_amplification_threshold = 8, int write_amplification_threshold = 32, uint64_t target_db_size = 68719476736 /* 64GB */); + +bool GetOptionsFromStrings( + const Options& base_options, + const std::unordered_map& options_map, + Options* new_options); + } // namespace rocksdb #endif // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_ diff --git a/util/options_helper.cc b/util/options_helper.cc new file mode 100644 index 000000000..a4d46ccb0 --- /dev/null +++ b/util/options_helper.cc @@ -0,0 +1,292 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "rocksdb/options.h" + +namespace rocksdb { + +namespace { +CompressionType ParseCompressionType(const std::string& type) { + if (type == "kNoCompression") { + return kNoCompression; + } else if (type == "kSnappyCompression") { + return kSnappyCompression; + } else if (type == "kZlibCompression") { + return kZlibCompression; + } else if (type == "kBZip2Compression") { + return kBZip2Compression; + } else if (type == "kLZ4Compression") { + return kLZ4Compression; + } else if (type == "kLZ4HCCompression") { + return kLZ4HCCompression; + } else { + throw "unknown compression type: " + type; + } + return kNoCompression; +} + +bool ParseBoolean(const std::string& type, const std::string& value) { + if (value == "true" || value == "1") { + return true; + } else if (value == "false" || value == "0") { + return false; + } else { + throw type; + } +} +uint32_t ParseInt(const std::string& value) { + return std::stoi(value); +} + +uint32_t ParseUint32(const std::string& value) { + return std::stoul(value); +} + +uint64_t ParseUint64(const std::string& value) { + return std::stoull(value); +} + +int64_t ParseInt64(const std::string& value) { + return std::stol(value); +} + +double ParseDouble(const std::string& value) { + return std::stod(value); +} + +CompactionStyle ParseCompactionStyle(const std::string& type) { + if (type == "kCompactionStyleLevel") { + return kCompactionStyleLevel; + } else if (type == "kCompactionStyleUniversal") { + return kCompactionStyleUniversal; + } else if (type == "kCompactionStyleFIFO") { + return kCompactionStyleFIFO; + } else { + throw "unknown compaction style: " + type; + } + return kCompactionStyleLevel; +} +} // anonymouse namespace + +bool GetOptionsFromStrings( + const Options& base_options, + const std::unordered_map& options_map, + Options* new_options) { + assert(new_options); + *new_options = base_options; + for (const auto& o : options_map) { + try { + if (o.first == "write_buffer_size") { + new_options->write_buffer_size = ParseInt64(o.second); + } else if (o.first == "max_write_buffer_number") { + new_options->max_write_buffer_number = ParseInt(o.second); + } else if (o.first == "min_write_buffer_number_to_merge") { + new_options->min_write_buffer_number_to_merge = ParseInt(o.second); + } else if (o.first == "compression") { + new_options->compression = ParseCompressionType(o.second); + } else if (o.first == "compression_per_level") { + new_options->compression_per_level.clear(); + size_t start = 0; + while (true) { + size_t end = o.second.find_first_of(':', start); + if (end == std::string::npos) { + new_options->compression_per_level.push_back( + ParseCompressionType(o.second.substr(start))); + break; + } else { + new_options->compression_per_level.push_back( + ParseCompressionType(o.second.substr(start, end - start))); + start = end + 1; + } + } + } else if (o.first == "compression_opts") { + size_t start = 0; + size_t end = o.second.find_first_of(':'); + if (end == std::string::npos) { + throw o.first; + } + new_options->compression_opts.window_bits = + ParseInt(o.second.substr(start, end - start)); + start = end + 1; + end = o.second.find_first_of(':', start); + if (end == std::string::npos) { + throw o.first; + } + new_options->compression_opts.level = + ParseInt(o.second.substr(start, end - start)); + start = end + 1; + if (start >= o.second.size()) { + throw o.first; + } + new_options->compression_opts.strategy = + ParseInt(o.second.substr(start, o.second.size() - start)); + } else if (o.first == "num_levels") { + new_options->num_levels = ParseInt(o.second); + } else if (o.first == "level0_file_num_compaction_trigger") { + new_options->level0_file_num_compaction_trigger = ParseInt(o.second); + } else if (o.first == "level0_slowdown_writes_trigger") { + new_options->level0_slowdown_writes_trigger = ParseInt(o.second); + } else if (o.first == "level0_stop_writes_trigger") { + new_options->level0_stop_writes_trigger = ParseInt(o.second); + } else if (o.first == "max_mem_compaction_level") { + new_options->max_mem_compaction_level = ParseInt(o.second); + } else if (o.first == "target_file_size_base") { + new_options->target_file_size_base = ParseInt(o.second); + } else if (o.first == "target_file_size_multiplier") { + new_options->target_file_size_multiplier = ParseInt(o.second); + } else if (o.first == "max_bytes_for_level_base") { + new_options->max_bytes_for_level_base = ParseUint64(o.second); + } else if (o.first == "max_bytes_for_level_multiplier") { + new_options->max_bytes_for_level_multiplier = ParseInt(o.second); + } else if (o.first == "max_bytes_for_level_multiplier_additional") { + new_options->max_bytes_for_level_multiplier_additional.clear(); + size_t start = 0; + while (true) { + size_t end = o.second.find_first_of(':', start); + if (end == std::string::npos) { + new_options->max_bytes_for_level_multiplier_additional.push_back( + ParseInt(o.second.substr(start))); + break; + } else { + new_options->max_bytes_for_level_multiplier_additional.push_back( + ParseInt(o.second.substr(start, end - start))); + start = end + 1; + } + } + } else if (o.first == "expanded_compaction_factor") { + new_options->expanded_compaction_factor = ParseInt(o.second); + } else if (o.first == "source_compaction_factor") { + new_options->source_compaction_factor = ParseInt(o.second); + } else if (o.first == "max_grandparent_overlap_factor") { + new_options->max_grandparent_overlap_factor = ParseInt(o.second); + } else if (o.first == "soft_rate_limit") { + new_options->soft_rate_limit = ParseDouble(o.second); + } else if (o.first == "hard_rate_limit") { + new_options->hard_rate_limit = ParseDouble(o.second); + } else if (o.first == "arena_block_size") { + new_options->arena_block_size = ParseInt64(o.second); + } else if (o.first == "disable_auto_compactions") { + new_options->disable_auto_compactions = ParseBoolean(o.first, o.second); + } else if (o.first == "purge_redundant_kvs_while_flush") { + new_options->purge_redundant_kvs_while_flush = + ParseBoolean(o.first, o.second); + } else if (o.first == "compaction_style") { + new_options->compaction_style = ParseCompactionStyle(o.second); + } else if (o.first == "verify_checksums_in_compaction") { + new_options->verify_checksums_in_compaction = + ParseBoolean(o.first, o.second); + } else if (o.first == "compaction_options_universal") { + // TODO(ljin): add support + throw o.first; + } else if (o.first == "compaction_options_fifo") { + new_options->compaction_options_fifo.max_table_files_size + = ParseUint64(o.second); + } else if (o.first == "filter_deletes") { + new_options->filter_deletes = ParseBoolean(o.first, o.second); + } else if (o.first == "max_sequential_skip_in_iterations") { + new_options->max_sequential_skip_in_iterations = ParseUint64(o.second); + } else if (o.first == "inplace_update_support") { + new_options->inplace_update_support = ParseBoolean(o.first, o.second); + } else if (o.first == "inplace_update_num_locks") { + new_options->inplace_update_num_locks = ParseInt64(o.second); + } else if (o.first == "memtable_prefix_bloom_bits") { + new_options->memtable_prefix_bloom_bits = stoul(o.second); + } else if (o.first == "memtable_prefix_bloom_probes") { + new_options->memtable_prefix_bloom_probes = stoul(o.second); + } else if (o.first == "memtable_prefix_bloom_huge_page_tlb_size") { + new_options->memtable_prefix_bloom_huge_page_tlb_size = + ParseInt64(o.second); + } else if (o.first == "bloom_locality") { + new_options->bloom_locality = ParseUint32(o.second); + } else if (o.first == "max_successive_merges") { + new_options->max_successive_merges = ParseInt64(o.second); + } else if (o.first == "min_partial_merge_operands") { + new_options->min_partial_merge_operands = ParseUint32(o.second); + } else if (o.first == "create_if_missing") { + new_options->create_if_missing = ParseBoolean(o.first, o.second); + } else if (o.first == "create_missing_column_families") { + new_options->create_missing_column_families = + ParseBoolean(o.first, o.second); + } else if (o.first == "error_if_exists") { + new_options->error_if_exists = ParseBoolean(o.first, o.second); + } else if (o.first == "paranoid_checks") { + new_options->paranoid_checks = ParseBoolean(o.first, o.second); + } else if (o.first == "max_open_files") { + new_options->max_open_files = ParseInt(o.second); + } else if (o.first == "max_total_wal_size") { + new_options->max_total_wal_size = ParseUint64(o.second); + } else if (o.first == "disable_data_sync") { + new_options->disableDataSync = ParseBoolean(o.first, o.second); + } else if (o.first == "use_fsync") { + new_options->use_fsync = ParseBoolean(o.first, o.second); + } else if (o.first == "db_paths") { + // TODO(ljin): add support + throw o.first; + } else if (o.first == "db_log_dir") { + new_options->db_log_dir = o.second; + } else if (o.first == "wal_dir") { + new_options->wal_dir = o.second; + } else if (o.first == "delete_obsolete_files_period_micros") { + new_options->delete_obsolete_files_period_micros = + ParseUint64(o.second); + } else if (o.first == "max_background_compactions") { + new_options->max_background_compactions = ParseInt(o.second); + } else if (o.first == "max_background_flushes") { + new_options->max_background_flushes = ParseInt(o.second); + } else if (o.first == "max_log_file_size") { + new_options->max_log_file_size = ParseInt64(o.second); + } else if (o.first == "log_file_time_to_roll") { + new_options->log_file_time_to_roll = ParseInt64(o.second); + } else if (o.first == "keep_log_file_num") { + new_options->keep_log_file_num = ParseInt64(o.second); + } else if (o.first == "max_manifest_file_size") { + new_options->max_manifest_file_size = ParseUint64(o.second); + } else if (o.first == "table_cache_numshardbits") { + new_options->table_cache_numshardbits = ParseInt(o.second); + } else if (o.first == "table_cache_remove_scan_count_limit") { + new_options->table_cache_remove_scan_count_limit = ParseInt(o.second); + } else if (o.first == "WAL_ttl_seconds") { + new_options->WAL_ttl_seconds = ParseUint64(o.second); + } else if (o.first == "WAL_size_limit_MB") { + new_options->WAL_size_limit_MB = ParseUint64(o.second); + } else if (o.first == "manifest_preallocation_size") { + new_options->manifest_preallocation_size = ParseInt64(o.second); + } else if (o.first == "allow_os_buffer") { + new_options->allow_os_buffer = ParseBoolean(o.first, o.second); + } else if (o.first == "allow_mmap_reads") { + new_options->allow_mmap_reads = ParseBoolean(o.first, o.second); + } else if (o.first == "allow_mmap_writes") { + new_options->allow_mmap_writes = ParseBoolean(o.first, o.second); + } else if (o.first == "is_fd_close_on_exec") { + new_options->is_fd_close_on_exec = ParseBoolean(o.first, o.second); + } else if (o.first == "skip_log_error_on_recovery") { + new_options->skip_log_error_on_recovery = + ParseBoolean(o.first, o.second); + } else if (o.first == "stats_dump_period_sec") { + new_options->stats_dump_period_sec = ParseUint32(o.second); + } else if (o.first == "advise_random_on_open") { + new_options->advise_random_on_open = ParseBoolean(o.first, o.second); + } else if (o.first == "use_adaptive_mutex") { + new_options->use_adaptive_mutex = ParseBoolean(o.first, o.second); + } else if (o.first == "allow_thread_local") { + new_options->allow_thread_local = ParseBoolean(o.first, o.second); + } else if (o.first == "bytes_per_sync") { + new_options->bytes_per_sync = ParseUint64(o.second); + } else { + return false; + } + } catch (std::exception) { + return false; + } + } + return true; +} + +} // namespace rocksdb diff --git a/util/options_test.cc b/util/options_test.cc index afe3795f9..c675cb87f 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -11,6 +11,7 @@ #define __STDC_FORMAT_MACROS #endif +#include #include #include @@ -75,6 +76,177 @@ TEST(OptionsTest, LooseCondition) { // Both tight amplifications PrintAndGetOptions(128 * 1024 * 1024, 4, 8); } + +TEST(OptionsTest, GetOptionsFromStringsTest) { + std::unordered_map options_map = { + {"write_buffer_size", "1"}, + {"max_write_buffer_number", "2"}, + {"min_write_buffer_number_to_merge", "3"}, + {"compression", "kSnappyCompression"}, + {"compression_per_level", "kNoCompression:" + "kSnappyCompression:" + "kZlibCompression:" + "kBZip2Compression:" + "kLZ4Compression:" + "kLZ4HCCompression"}, + {"compression_opts", "4:5:6"}, + {"num_levels", "7"}, + {"level0_file_num_compaction_trigger", "8"}, + {"level0_slowdown_writes_trigger", "9"}, + {"level0_stop_writes_trigger", "10"}, + {"max_mem_compaction_level", "11"}, + {"target_file_size_base", "12"}, + {"target_file_size_multiplier", "13"}, + {"max_bytes_for_level_base", "14"}, + {"max_bytes_for_level_multiplier", "15"}, + {"max_bytes_for_level_multiplier_additional", "16:17:18"}, + {"expanded_compaction_factor", "19"}, + {"source_compaction_factor", "20"}, + {"max_grandparent_overlap_factor", "21"}, + {"soft_rate_limit", "1.1"}, + {"hard_rate_limit", "2.1"}, + {"arena_block_size", "22"}, + {"disable_auto_compactions", "true"}, + {"purge_redundant_kvs_while_flush", "1"}, + {"compaction_style", "kCompactionStyleLevel"}, + {"verify_checksums_in_compaction", "false"}, + {"compaction_options_fifo", "23"}, + {"filter_deletes", "0"}, + {"max_sequential_skip_in_iterations", "24"}, + {"inplace_update_support", "true"}, + {"inplace_update_num_locks", "25"}, + {"memtable_prefix_bloom_bits", "26"}, + {"memtable_prefix_bloom_probes", "27"}, + {"memtable_prefix_bloom_huge_page_tlb_size", "28"}, + {"bloom_locality", "29"}, + {"max_successive_merges", "30"}, + {"min_partial_merge_operands", "31"}, + {"create_if_missing", "false"}, + {"create_missing_column_families", "true"}, + {"error_if_exists", "false"}, + {"paranoid_checks", "true"}, + {"max_open_files", "32"}, + {"max_total_wal_size", "33"}, + {"disable_data_sync", "false"}, + {"use_fsync", "true"}, + {"db_log_dir", "/db_log_dir"}, + {"wal_dir", "/wal_dir"}, + {"delete_obsolete_files_period_micros", "34"}, + {"max_background_compactions", "35"}, + {"max_background_flushes", "36"}, + {"max_log_file_size", "37"}, + {"log_file_time_to_roll", "38"}, + {"keep_log_file_num", "39"}, + {"max_manifest_file_size", "40"}, + {"table_cache_numshardbits", "41"}, + {"table_cache_remove_scan_count_limit", "42"}, + {"WAL_ttl_seconds", "43"}, + {"WAL_size_limit_MB", "44"}, + {"manifest_preallocation_size", "45"}, + {"allow_os_buffer", "false"}, + {"allow_mmap_reads", "true"}, + {"allow_mmap_writes", "false"}, + {"is_fd_close_on_exec", "true"}, + {"skip_log_error_on_recovery", "false"}, + {"stats_dump_period_sec", "46"}, + {"advise_random_on_open", "true"}, + {"use_adaptive_mutex", "false"}, + {"allow_thread_local", "true"}, + {"bytes_per_sync", "47"}, + }; + + Options base_opt; + Options new_opt; + ASSERT_TRUE(GetOptionsFromStrings(base_opt, options_map, &new_opt)); + ASSERT_EQ(new_opt.write_buffer_size, 1); + ASSERT_EQ(new_opt.max_write_buffer_number, 2); + ASSERT_EQ(new_opt.min_write_buffer_number_to_merge, 3); + ASSERT_EQ(new_opt.compression, kSnappyCompression); + ASSERT_EQ(new_opt.compression_per_level.size(), 6); + ASSERT_EQ(new_opt.compression_per_level[0], kNoCompression); + ASSERT_EQ(new_opt.compression_per_level[1], kSnappyCompression); + ASSERT_EQ(new_opt.compression_per_level[2], kZlibCompression); + ASSERT_EQ(new_opt.compression_per_level[3], kBZip2Compression); + ASSERT_EQ(new_opt.compression_per_level[4], kLZ4Compression); + ASSERT_EQ(new_opt.compression_per_level[5], kLZ4HCCompression); + ASSERT_EQ(new_opt.compression_opts.window_bits, 4); + ASSERT_EQ(new_opt.compression_opts.level, 5); + ASSERT_EQ(new_opt.compression_opts.strategy, 6); + ASSERT_EQ(new_opt.num_levels, 7); + ASSERT_EQ(new_opt.level0_file_num_compaction_trigger, 8); + ASSERT_EQ(new_opt.level0_slowdown_writes_trigger, 9); + ASSERT_EQ(new_opt.level0_stop_writes_trigger, 10); + ASSERT_EQ(new_opt.max_mem_compaction_level, 11); + ASSERT_EQ(new_opt.target_file_size_base, 12); + ASSERT_EQ(new_opt.target_file_size_multiplier, 13); + ASSERT_EQ(new_opt.max_bytes_for_level_base, 14); + ASSERT_EQ(new_opt.max_bytes_for_level_multiplier, 15); + ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional.size(), 3); + ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional[0], 16); + ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional[1], 17); + ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional[2], 18); + ASSERT_EQ(new_opt.expanded_compaction_factor, 19); + ASSERT_EQ(new_opt.source_compaction_factor, 20); + ASSERT_EQ(new_opt.max_grandparent_overlap_factor, 21); + ASSERT_EQ(new_opt.soft_rate_limit, 1.1); + ASSERT_EQ(new_opt.hard_rate_limit, 2.1); + ASSERT_EQ(new_opt.arena_block_size, 22); + ASSERT_EQ(new_opt.disable_auto_compactions, true); + ASSERT_EQ(new_opt.purge_redundant_kvs_while_flush, true); + ASSERT_EQ(new_opt.compaction_style, kCompactionStyleLevel); + ASSERT_EQ(new_opt.verify_checksums_in_compaction, false); + ASSERT_EQ(new_opt.compaction_options_fifo.max_table_files_size, 23); + ASSERT_EQ(new_opt.filter_deletes, false); + ASSERT_EQ(new_opt.max_sequential_skip_in_iterations, 24); + ASSERT_EQ(new_opt.inplace_update_support, true); + ASSERT_EQ(new_opt.inplace_update_num_locks, 25); + ASSERT_EQ(new_opt.memtable_prefix_bloom_bits, 26); + ASSERT_EQ(new_opt.memtable_prefix_bloom_probes, 27); + ASSERT_EQ(new_opt.memtable_prefix_bloom_huge_page_tlb_size, 28); + ASSERT_EQ(new_opt.bloom_locality, 29); + ASSERT_EQ(new_opt.max_successive_merges, 30); + ASSERT_EQ(new_opt.min_partial_merge_operands, 31); + ASSERT_EQ(new_opt.create_if_missing, false); + ASSERT_EQ(new_opt.create_missing_column_families, true); + ASSERT_EQ(new_opt.error_if_exists, false); + ASSERT_EQ(new_opt.paranoid_checks, true); + ASSERT_EQ(new_opt.max_open_files, 32); + ASSERT_EQ(new_opt.max_total_wal_size, 33); + ASSERT_EQ(new_opt.disableDataSync, false); + ASSERT_EQ(new_opt.use_fsync, true); + ASSERT_EQ(new_opt.db_log_dir, "/db_log_dir"); + ASSERT_EQ(new_opt.wal_dir, "/wal_dir"); + ASSERT_EQ(new_opt.delete_obsolete_files_period_micros, 34); + ASSERT_EQ(new_opt.max_background_compactions, 35); + ASSERT_EQ(new_opt.max_background_flushes, 36); + ASSERT_EQ(new_opt.max_log_file_size, 37); + ASSERT_EQ(new_opt.log_file_time_to_roll, 38); + ASSERT_EQ(new_opt.keep_log_file_num, 39); + ASSERT_EQ(new_opt.max_manifest_file_size, 40); + ASSERT_EQ(new_opt.table_cache_numshardbits, 41); + ASSERT_EQ(new_opt.table_cache_remove_scan_count_limit, 42); + ASSERT_EQ(new_opt.WAL_ttl_seconds, 43); + ASSERT_EQ(new_opt.WAL_size_limit_MB, 44); + ASSERT_EQ(new_opt.manifest_preallocation_size, 45); + ASSERT_EQ(new_opt.allow_os_buffer, false); + ASSERT_EQ(new_opt.allow_mmap_reads, true); + ASSERT_EQ(new_opt.allow_mmap_writes, false); + ASSERT_EQ(new_opt.is_fd_close_on_exec, true); + ASSERT_EQ(new_opt.skip_log_error_on_recovery, false); + ASSERT_EQ(new_opt.stats_dump_period_sec, 46); + ASSERT_EQ(new_opt.advise_random_on_open, true); + ASSERT_EQ(new_opt.use_adaptive_mutex, false); + ASSERT_EQ(new_opt.allow_thread_local, true); + ASSERT_EQ(new_opt.bytes_per_sync, 47); + + options_map["write_buffer_size"] = "hello"; + ASSERT_TRUE(!GetOptionsFromStrings(base_opt, options_map, &new_opt)); + options_map["write_buffer_size"] = "1"; + ASSERT_TRUE(GetOptionsFromStrings(base_opt, options_map, &new_opt)); + options_map["unknown_option"] = "1"; + ASSERT_TRUE(!GetOptionsFromStrings(base_opt, options_map, &new_opt)); +} + } // namespace rocksdb int main(int argc, char** argv) {