From 618bf638aabce21262228509e9f99c1c13de2b57 Mon Sep 17 00:00:00 2001 From: mrambacher Date: Tue, 28 Apr 2020 18:02:11 -0700 Subject: [PATCH] Add Functions to OptionTypeInfo (#6422) Summary: Added functions for parsing, serializing, and comparing elements to OptionTypeInfo. These functions allow all of the special cases that could not be handled directly in the map of OptionTypeInfo to be moved into the map. Using these functions, every type can be handled via the map rather than special cased. By adding these functions, the code for handling options can become more standardized (fewer special cases) and (eventually) handled completely by common classes. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6422 Test Plan: pass make check Reviewed By: siying Differential Revision: D21269005 Pulled By: zhichao-cao fbshipit-source-id: 9ba71c721a38ebf9ee88259d60bd81b3282b9077 --- CMakeLists.txt | 2 +- TARGETS | 2 +- cache/cache.cc | 41 ++ db/db_options_test.cc | 4 +- include/rocksdb/cache.h | 16 + include/rocksdb/convenience.h | 12 + include/rocksdb/filter_policy.h | 15 + options/cf_options.cc | 302 ++++++++---- options/db_options.cc | 32 ++ options/options_helper.cc | 460 +++++++++++------- options/options_helper.h | 4 - options/options_parser.cc | 265 +++------- options/options_parser.h | 5 - options/options_sanity_check.cc | 39 -- options/options_sanity_check.h | 45 -- options/options_test.cc | 276 ++++++++++- options/options_type.h | 168 ++++++- src.mk | 2 +- .../block_based/block_based_table_factory.cc | 151 +++--- table/block_based/filter_policy.cc | 30 ++ table/plain/plain_table_factory.cc | 12 +- 21 files changed, 1210 insertions(+), 673 deletions(-) create mode 100644 cache/cache.cc delete mode 100644 options/options_sanity_check.cc delete mode 100644 options/options_sanity_check.h diff --git a/CMakeLists.txt b/CMakeLists.txt index bf99be5e0..302575cb7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -518,6 +518,7 @@ find_package(Threads REQUIRED) # Main library source code set(SOURCES + cache/cache.cc cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc @@ -632,7 +633,6 @@ set(SOURCES options/options.cc options/options_helper.cc options/options_parser.cc - options/options_sanity_check.cc port/stack_trace.cc table/adaptive/adaptive_table_factory.cc table/block_based/binary_search_index_reader.cc diff --git a/TARGETS b/TARGETS index 1200c0023..57f2b647b 100644 --- a/TARGETS +++ b/TARGETS @@ -112,6 +112,7 @@ ROCKSDB_OS_DEPS += ([( cpp_library( name = "rocksdb_lib", srcs = [ + "cache/cache.cc", "cache/clock_cache.cc", "cache/lru_cache.cc", "cache/sharded_cache.cc", @@ -230,7 +231,6 @@ cpp_library( "options/options.cc", "options/options_helper.cc", "options/options_parser.cc", - "options/options_sanity_check.cc", "port/port_posix.cc", "port/stack_trace.cc", "table/adaptive/adaptive_table_factory.cc", diff --git a/cache/cache.cc b/cache/cache.cc new file mode 100644 index 000000000..f4a480533 --- /dev/null +++ b/cache/cache.cc @@ -0,0 +1,41 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/cache.h" + +#include "cache/lru_cache.h" +#include "options/options_helper.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +Status Cache::CreateFromString(const ConfigOptions& /*opts*/, + const std::string& value, + std::shared_ptr* result) { + Status status; + std::shared_ptr cache; + if (value.find('=') == std::string::npos) { + cache = NewLRUCache(ParseSizeT(value)); + } else { +#ifndef ROCKSDB_LITE + LRUCacheOptions cache_opts; + if (!ParseOptionHelper(reinterpret_cast(&cache_opts), + OptionType::kLRUCacheOptions, value)) { + status = Status::InvalidArgument("Invalid cache options"); + } + cache = NewLRUCache(cache_opts); +#else + status = Status::NotSupported("Cannot load cache in LITE mode ", value); +#endif //! ROCKSDB_LITE + } + if (status.ok()) { + result->swap(cache); + } + return status; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/db/db_options_test.cc b/db/db_options_test.cc index a884ac58e..23be9cda3 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -40,7 +40,7 @@ class DBOptionsTest : public DBTestBase { StringToMap(options_str, &options_map); std::unordered_map mutable_map; for (const auto opt : db_options_type_info) { - if (opt.second.IsMutable() && !opt.second.IsDeprecated()) { + if (opt.second.IsMutable() && opt.second.ShouldSerialize()) { mutable_map[opt.first] = options_map[opt.first]; } } @@ -58,7 +58,7 @@ class DBOptionsTest : public DBTestBase { StringToMap(options_str, &options_map); std::unordered_map mutable_map; for (const auto opt : cf_options_type_info) { - if (opt.second.IsMutable() && !opt.second.IsDeprecated()) { + if (opt.second.IsMutable() && opt.second.ShouldSerialize()) { mutable_map[opt.first] = options_map[opt.first]; } } diff --git a/include/rocksdb/cache.h b/include/rocksdb/cache.h index 77ddf525d..e4c404333 100644 --- a/include/rocksdb/cache.h +++ b/include/rocksdb/cache.h @@ -33,6 +33,7 @@ namespace ROCKSDB_NAMESPACE { class Cache; +struct ConfigOptions; extern const bool kDefaultToAdaptiveMutex; @@ -142,6 +143,21 @@ class Cache { Cache(const Cache&) = delete; Cache& operator=(const Cache&) = delete; + // Creates a new Cache based on the input value string and returns the result. + // Currently, this method can be used to create LRUCaches only + // @param config_options + // @param value The value might be: + // - an old-style cache ("1M") -- equivalent to NewLRUCache(1024*102( + // - Name-value option pairs -- "capacity=1M; num_shard_bits=4; + // For the LRUCache, the values are defined in LRUCacheOptions. + // @param result The new Cache object + // @return OK if the cache was sucessfully created + // @return NotFound if an invalid name was specified in the value + // @return InvalidArgument if either the options were not valid + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result); + // Destroys all existing entries by calling the "deleter" // function that was passed via the Insert() function. // diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index d8843cb76..41bbd8469 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -226,6 +226,12 @@ struct ConfigOptions { // instead of resulting in an unknown-option error. // @return Status::OK() on success. Otherwise, a non-ok status indicating // error will be returned, and "new_options" will be set to "base_options". +// @return Status::NotFound means the one (or more) of the option name in +// the opts_map is not valid for this option +// @return Status::NotSupported means we do not know how to parse one of the +// value for this option +// @return Status::InvalidArgument means the one of the option values is not +// valid for this option. Status GetColumnFamilyOptionsFromMap( const ConfigOptions& config_options, const ColumnFamilyOptions& base_options, @@ -268,6 +274,12 @@ Status GetColumnFamilyOptionsFromMap( // instead of resulting in an unknown-option error. // @return Status::OK() on success. Otherwise, a non-ok status indicating // error will be returned, and "new_options" will be set to "base_options". +// @return Status::NotFound means the one (or more) of the option name in +// the opts_map is not valid for this option +// @return Status::NotSupported means we do not know how to parse one of the +// value for this option +// @return Status::InvalidArgument means the one of the option values is not +// valid for this option. Status GetDBOptionsFromMap( const ConfigOptions& cfg_options, const DBOptions& base_options, const std::unordered_map& opts_map, diff --git a/include/rocksdb/filter_policy.h b/include/rocksdb/filter_policy.h index 03d6471cf..3cd85a226 100644 --- a/include/rocksdb/filter_policy.h +++ b/include/rocksdb/filter_policy.h @@ -20,17 +20,20 @@ #pragma once #include + #include #include #include #include #include "rocksdb/advanced_options.h" +#include "rocksdb/status.h" namespace ROCKSDB_NAMESPACE { class Slice; struct BlockBasedTableOptions; +struct ConfigOptions; // A class that takes a bunch of keys, then generates filter class FilterBitsBuilder { @@ -125,6 +128,18 @@ class FilterPolicy { public: virtual ~FilterPolicy(); + // Creates a new FilterPolicy based on the input value string and returns the + // result The value might be an ID, and ID with properties, or an old-style + // policy string. + // The value describes the FilterPolicy being created. + // For BloomFilters, value may be a ":"-delimited value of the form: + // "bloomfilter:[bits_per_key]:[use_block_based_builder]", + // e.g. ""bloomfilter:4:true" + // The above string is equivalent to calling NewBloomFilterPolicy(4, true). + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result); + // Return the name of this policy. Note that if the filter encoding // changes in an incompatible way, the name returned by this method // must be changed. Otherwise, old incompatible filters may be diff --git a/options/cf_options.cc b/options/cf_options.cc index ec1267030..84b52e0d1 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -47,8 +47,79 @@ int offset_of(T1 AdvancedColumnFamilyOptions::*member) { size_t(&OptionsHelper::dummy_cf_options)); } -const std::string kNameComparator = "comparator"; -const std::string kNameMergeOperator = "merge_operator"; +static Status ParseCompressionOptions(const std::string& value, + const std::string& name, + CompressionOptions& compression_opts) { + size_t start = 0; + size_t end = value.find(':'); + if (end == std::string::npos) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + compression_opts.window_bits = ParseInt(value.substr(start, end - start)); + start = end + 1; + end = value.find(':', start); + if (end == std::string::npos) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + compression_opts.level = ParseInt(value.substr(start, end - start)); + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + end = value.find(':', start); + compression_opts.strategy = + ParseInt(value.substr(start, value.size() - start)); + // max_dict_bytes is optional for backwards compatibility + if (end != std::string::npos) { + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.max_dict_bytes = + ParseInt(value.substr(start, value.size() - start)); + end = value.find(':', start); + } + // zstd_max_train_bytes is optional for backwards compatibility + if (end != std::string::npos) { + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.zstd_max_train_bytes = + ParseInt(value.substr(start, value.size() - start)); + end = value.find(':', start); + } + // parallel_threads is optional for backwards compatibility + if (end != std::string::npos) { + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.parallel_threads = + ParseInt(value.substr(start, value.size() - start)); + end = value.find(':', start); + } + // enabled is optional for backwards compatibility + if (end != std::string::npos) { + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.enabled = + ParseBoolean("", value.substr(start, value.size() - start)); + } + return Status::OK(); +} + +const std::string kOptNameBMCompOpts = "bottommost_compression_opts"; +const std::string kOptNameCompOpts = "compression_opts"; std::unordered_map OptionsHelper::cf_options_type_info = { @@ -280,9 +351,22 @@ std::unordered_map OptionType::kCompressionType, OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct MutableCFOptions, bottommost_compression)}}, - {kNameComparator, + {"comparator", {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, - OptionVerificationType::kByName, OptionTypeFlags::kNone, 0}}, + OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose, 0, + // Parses the string and sets the corresponding comparator + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto old_comparator = reinterpret_cast(addr); + const Comparator* new_comparator = *old_comparator; + Status status = ObjectRegistry::NewInstance()->NewStaticObject( + value, &new_comparator); + if (status.ok()) { + *old_comparator = new_comparator; + return status; + } + return Status::OK(); + }}}, {"prefix_extractor", {offset_of(&ColumnFamilyOptions::prefix_extractor), OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, @@ -297,10 +381,74 @@ std::unordered_map {offset_of(&ColumnFamilyOptions::memtable_factory), OptionType::kMemTableRepFactory, OptionVerificationType::kByName, OptionTypeFlags::kNone, 0}}, + {"memtable", + {offset_of(&ColumnFamilyOptions::memtable_factory), + OptionType::kMemTableRepFactory, OptionVerificationType::kAlias, + OptionTypeFlags::kNone, 0, + // Parses the value string and updates the memtable_factory + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + std::unique_ptr new_mem_factory; + Status s = GetMemTableRepFactoryFromString(value, &new_mem_factory); + if (s.ok()) { + auto memtable_factory = + reinterpret_cast*>(addr); + memtable_factory->reset(new_mem_factory.release()); + } + return s; + }}}, {"table_factory", {offset_of(&ColumnFamilyOptions::table_factory), OptionType::kTableFactory, OptionVerificationType::kByName, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kCompareLoose, 0}}, + {"block_based_table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kTableFactory, OptionVerificationType::kAlias, + OptionTypeFlags::kCompareLoose, 0, + // Parses the input value and creates a BlockBasedTableFactory + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + // Nested options + auto old_table_factory = + reinterpret_cast*>(addr); + BlockBasedTableOptions table_opts, base_opts; + BlockBasedTableFactory* block_based_table_factory = + static_cast_with_check( + old_table_factory->get()); + if (block_based_table_factory != nullptr) { + base_opts = block_based_table_factory->table_options(); + } + Status s = GetBlockBasedTableOptionsFromString(base_opts, value, + &table_opts); + if (s.ok()) { + old_table_factory->reset(NewBlockBasedTableFactory(table_opts)); + } + return s; + }}}, + {"plain_table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kTableFactory, OptionVerificationType::kAlias, + OptionTypeFlags::kCompareLoose, 0, + // Parses the input value and creates a PlainTableFactory + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + // Nested options + auto old_table_factory = + reinterpret_cast*>(addr); + PlainTableOptions table_opts, base_opts; + PlainTableFactory* plain_table_factory = + static_cast_with_check( + old_table_factory->get()); + if (plain_table_factory != nullptr) { + base_opts = plain_table_factory->table_options(); + } + Status s = + GetPlainTableOptionsFromString(base_opts, value, &table_opts); + if (s.ok()) { + old_table_factory->reset(NewPlainTableFactory(table_opts)); + } + return s; + }}}, {"compaction_filter", {offset_of(&ColumnFamilyOptions::compaction_filter), OptionType::kCompactionFilter, OptionVerificationType::kByName, @@ -309,11 +457,19 @@ std::unordered_map {offset_of(&ColumnFamilyOptions::compaction_filter_factory), OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, OptionTypeFlags::kNone, 0}}, - {kNameMergeOperator, + {"merge_operator", {offset_of(&ColumnFamilyOptions::merge_operator), OptionType::kMergeOperator, - OptionVerificationType::kByNameAllowFromNull, OptionTypeFlags::kNone, - 0}}, + OptionVerificationType::kByNameAllowFromNull, + OptionTypeFlags::kCompareLoose, 0, + // Parses the input value as a MergeOperator, updating the value + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto mop = reinterpret_cast*>(addr); + ObjectRegistry::NewInstance()->NewSharedObject(value, + mop); + return Status::OK(); + }}}, {"compaction_style", {offset_of(&ColumnFamilyOptions::compaction_style), OptionType::kCompactionStyle, OptionVerificationType::kNormal, @@ -345,7 +501,36 @@ std::unordered_map {offset_of(&ColumnFamilyOptions::sample_for_compression), OptionType::kUInt64T, OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, sample_for_compression)}}}; + offsetof(struct MutableCFOptions, sample_for_compression)}}, + // The following properties were handled as special cases in ParseOption + // This means that the properties could be read from the options file + // but never written to the file or compared to each other. + {kOptNameCompOpts, + {offset_of(&ColumnFamilyOptions::compression_opts), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever | + OptionTypeFlags::kMutable), + offsetof(struct MutableCFOptions, compression_opts), + // Parses the value as a CompressionOptions + [](const ConfigOptions& /*opts*/, const std::string& name, + const std::string& value, char* addr) { + auto* compression = reinterpret_cast(addr); + return ParseCompressionOptions(value, name, *compression); + }}}, + {kOptNameBMCompOpts, + {offset_of(&ColumnFamilyOptions::bottommost_compression_opts), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever | + OptionTypeFlags::kMutable), + offsetof(struct MutableCFOptions, bottommost_compression_opts), + // Parses the value as a CompressionOptions + [](const ConfigOptions& /*opts*/, const std::string& name, + const std::string& value, char* addr) { + auto* compression = reinterpret_cast(addr); + return ParseCompressionOptions(value, name, *compression); + }}}, + // End special case properties +}; Status ParseColumnFamilyOption(const ConfigOptions& config_options, const std::string& name, @@ -355,104 +540,19 @@ Status ParseColumnFamilyOption(const ConfigOptions& config_options, ? UnescapeOptionString(org_value) : org_value; try { - if (name == "block_based_table_factory") { - // Nested options - BlockBasedTableOptions table_opt, base_table_options; - BlockBasedTableFactory* block_based_table_factory = - static_cast_with_check( - new_options->table_factory.get()); - if (block_based_table_factory != nullptr) { - base_table_options = block_based_table_factory->table_options(); - } - Status table_opt_s = GetBlockBasedTableOptionsFromString( - base_table_options, value, &table_opt); - if (!table_opt_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); - } else if (name == "plain_table_factory") { - // Nested options - PlainTableOptions table_opt, base_table_options; - PlainTableFactory* plain_table_factory = - static_cast_with_check( - new_options->table_factory.get()); - if (plain_table_factory != nullptr) { - base_table_options = plain_table_factory->table_options(); - } - Status table_opt_s = - GetPlainTableOptionsFromString(base_table_options, value, &table_opt); - if (!table_opt_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->table_factory.reset(NewPlainTableFactory(table_opt)); - } else if (name == "memtable") { - std::unique_ptr new_mem_factory; - Status mem_factory_s = - GetMemTableRepFactoryFromString(value, &new_mem_factory); - if (!mem_factory_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->memtable_factory.reset(new_mem_factory.release()); - } else if (name == "bottommost_compression_opts") { - Status s = ParseCompressionOptions( - value, name, new_options->bottommost_compression_opts); - if (!s.ok()) { - return s; - } - } else if (name == "compression_opts") { - Status s = - ParseCompressionOptions(value, name, new_options->compression_opts); - if (!s.ok()) { - return s; - } + auto iter = cf_options_type_info.find(name); + if (iter == cf_options_type_info.end()) { + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); } else { - if (name == kNameComparator) { - // Try to get comparator from object registry first. - // Only support static comparator for now. - Status status = ObjectRegistry::NewInstance()->NewStaticObject( - value, &new_options->comparator); - if (status.ok()) { - return status; - } - } else if (name == kNameMergeOperator) { - // Try to get merge operator from object registry first. - std::shared_ptr mo; - Status status = - ObjectRegistry::NewInstance()->NewSharedObject( - value, &new_options->merge_operator); - // Only support static comparator for now. - if (status.ok()) { - return status; - } - } - - auto iter = cf_options_type_info.find(name); - if (iter == cf_options_type_info.end()) { - return Status::InvalidArgument( - "Unable to parse the specified CF option " + name); - } - const auto& opt_info = iter->second; - if (opt_info.IsDeprecated() || - ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.offset, - opt_info.type, value)) { - return Status::OK(); - } else if (opt_info.IsByName()) { - return Status::NotSupported("Deserializing the specified CF option " + - name + " is not supported"); - } else { - return Status::InvalidArgument( - "Unable to parse the specified CF option " + name); - } + return iter->second.ParseOption( + config_options, name, value, + reinterpret_cast(new_options) + iter->second.offset); } } catch (const std::exception&) { return Status::InvalidArgument("unable to parse the specified option " + name); } - return Status::OK(); } #endif // ROCKSDB_LITE diff --git a/options/db_options.cc b/options/db_options.cc index d6b95b5c3..f77cb7b9e 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -14,6 +14,7 @@ #include "rocksdb/cache.h" #include "rocksdb/env.h" #include "rocksdb/file_system.h" +#include "rocksdb/rate_limiter.h" #include "rocksdb/sst_file_manager.h" #include "rocksdb/wal_filter.h" @@ -332,6 +333,37 @@ std::unordered_map {offsetof(struct DBOptions, best_efforts_recovery), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + // The following properties were handled as special cases in ParseOption + // This means that the properties could be read from the options file + // but never written to the file or compared to each other. + {"rate_limiter_bytes_per_sec", + {offsetof(struct DBOptions, rate_limiter), OptionType::kUnknown, + OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0, + // Parse the input value as a RateLimiter + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto limiter = + reinterpret_cast*>(addr); + limiter->reset(NewGenericRateLimiter( + static_cast(ParseUint64(value)))); + return Status::OK(); + }}}, + {"env", + {offsetof(struct DBOptions, env), OptionType::kUnknown, + OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0, + // Parse the input value as an Env + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto old_env = reinterpret_cast(addr); // Get the old value + Env* new_env = *old_env; // Set new to old + Status s = Env::LoadEnv(value, &new_env); // Update new value + if (s.ok()) { // It worked + *old_env = new_env; // Update the old one + } + return s; + }}}, }; #endif // ROCKSDB_LITE diff --git a/options/options_helper.cc b/options/options_helper.cc index e310ff903..63e6f6766 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -272,10 +272,6 @@ std::vector GetSupportedCompressions() { #ifndef ROCKSDB_LITE -const std::string kNameEnv = "env"; -const std::string kOptNameBMCompOpts = "bottommost_compression_opts"; -const std::string kOptNameCompOpts = "compression_opts"; - namespace { template bool ParseEnum(const std::unordered_map& type_map, @@ -382,7 +378,8 @@ static bool SerializeStruct( } static bool ParseSingleStructOption( - const std::string& opt_val_str, void* options, + const ConfigOptions& config_options, const std::string& opt_val_str, + void* options, const std::unordered_map& type_info_map) { size_t end = opt_val_str.find('='); std::string key = opt_val_str.substr(0, end); @@ -392,20 +389,17 @@ static bool ParseSingleStructOption( return false; } const auto& opt_info = iter->second; - if (opt_info.IsDeprecated()) { - // Should also skip deprecated sub-options such as - // fifo_compaction_options_type_info.ttl - return true; - } - return ParseOptionHelper( - reinterpret_cast(options) + opt_info.mutable_offset, opt_info.type, - value); + Status s = opt_info.ParseOption( + config_options, key, value, + reinterpret_cast(options) + opt_info.mutable_offset); + return s.ok(); } static bool ParseStructOptions( const std::string& opt_str, void* options, const std::unordered_map& type_info_map) { assert(!opt_str.empty()); + ConfigOptions config_options; size_t start = 0; if (opt_str[0] == '{') { @@ -417,8 +411,8 @@ static bool ParseStructOptions( } size_t end = opt_str.find(';', start); size_t len = (end == std::string::npos) ? end : end - start; - if (!ParseSingleStructOption(opt_str.substr(start, len), options, - type_info_map)) { + if (!ParseSingleStructOption(config_options, opt_str.substr(start, len), + options, type_info_map)) { return false; } start = (end == std::string::npos) ? end : end + 1; @@ -792,129 +786,34 @@ bool SerializeSingleOptionHelper(const char* opt_address, return true; } -Status ParseCompressionOptions(const std::string& value, - const std::string& name, - CompressionOptions& compression_opts) { - size_t start = 0; - size_t end = value.find(':'); - if (end == std::string::npos) { - return Status::InvalidArgument("unable to parse the specified CF option " + - name); - } - compression_opts.window_bits = ParseInt(value.substr(start, end - start)); - start = end + 1; - end = value.find(':', start); - if (end == std::string::npos) { - return Status::InvalidArgument("unable to parse the specified CF option " + - name); - } - compression_opts.level = ParseInt(value.substr(start, end - start)); - start = end + 1; - if (start >= value.size()) { - return Status::InvalidArgument("unable to parse the specified CF option " + - name); - } - end = value.find(':', start); - compression_opts.strategy = - ParseInt(value.substr(start, value.size() - start)); - // max_dict_bytes is optional for backwards compatibility - if (end != std::string::npos) { - start = end + 1; - if (start >= value.size()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - compression_opts.max_dict_bytes = - ParseInt(value.substr(start, value.size() - start)); - end = value.find(':', start); - } - // zstd_max_train_bytes is optional for backwards compatibility - if (end != std::string::npos) { - start = end + 1; - if (start >= value.size()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - compression_opts.zstd_max_train_bytes = - ParseInt(value.substr(start, value.size() - start)); - end = value.find(':', start); - } - // parallel_threads is optional for backwards compatibility - if (end != std::string::npos) { - start = end + 1; - if (start >= value.size()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - compression_opts.parallel_threads = - ParseInt(value.substr(start, value.size() - start)); - end = value.find(':', start); - } - // enabled is optional for backwards compatibility - if (end != std::string::npos) { - start = end + 1; - if (start >= value.size()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - compression_opts.enabled = - ParseBoolean("", value.substr(start, value.size() - start)); - } - return Status::OK(); -} - Status GetMutableOptionsFromStrings( const MutableCFOptions& base_options, const std::unordered_map& options_map, Logger* info_log, MutableCFOptions* new_options) { assert(new_options); *new_options = base_options; + ConfigOptions config_options; for (const auto& o : options_map) { - auto& option_name = o.first; - auto& option_value = o.second; - - try { - if (option_name == kOptNameBMCompOpts) { - Status s = - ParseCompressionOptions(option_value, option_name, - new_options->bottommost_compression_opts); - if (!s.ok()) { - return s; - } - } else if (option_name == kOptNameCompOpts) { - Status s = ParseCompressionOptions(option_value, option_name, - new_options->compression_opts); - if (!s.ok()) { - return s; - } - } else { - auto iter = cf_options_type_info.find(option_name); - if (iter == cf_options_type_info.end()) { - return Status::InvalidArgument("Unrecognized option: " + option_name); - } - const auto& opt_info = iter->second; - if (!opt_info.IsMutable()) { - return Status::InvalidArgument("Option not changeable: " + - option_name); - } - if (opt_info.IsDeprecated()) { - // log warning when user tries to set a deprecated option but don't - // fail the call for compatibility. - ROCKS_LOG_WARN(info_log, - "%s is a deprecated option and cannot be set", - option_name.c_str()); - continue; - } - bool is_ok = ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.mutable_offset, - opt_info.type, option_value); - if (!is_ok) { - return Status::InvalidArgument("Error parsing " + option_name); - } - } - } catch (std::exception& e) { - return Status::InvalidArgument("Error parsing " + option_name + ":" + - std::string(e.what())); + auto iter = cf_options_type_info.find(o.first); + if (iter == cf_options_type_info.end()) { + return Status::InvalidArgument("Unrecognized option: " + o.first); + } + const auto& opt_info = iter->second; + if (!opt_info.IsMutable()) { + return Status::InvalidArgument("Option not changeable: " + o.first); + } + if (opt_info.IsDeprecated()) { + // log warning when user tries to set a deprecated option but don't fail + // the call for compatibility. + ROCKS_LOG_WARN(info_log, "%s is a deprecated option and cannot be set", + o.first.c_str()); + continue; + } + Status s = opt_info.ParseOption( + config_options, o.first, o.second, + reinterpret_cast(new_options) + opt_info.mutable_offset); + if (!s.ok()) { + return s; } } return Status::OK(); @@ -926,6 +825,8 @@ Status GetMutableDBOptionsFromStrings( MutableDBOptions* new_options) { assert(new_options); *new_options = base_options; + ConfigOptions config_options; + for (const auto& o : options_map) { try { auto iter = db_options_type_info.find(o.first); @@ -936,11 +837,11 @@ Status GetMutableDBOptionsFromStrings( if (!opt_info.IsMutable()) { return Status::InvalidArgument("Option not changeable: " + o.first); } - bool is_ok = ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.mutable_offset, - opt_info.type, o.second); - if (!is_ok) { - return Status::InvalidArgument("Error parsing " + o.first); + Status s = opt_info.ParseOption( + config_options, o.first, o.second, + reinterpret_cast(new_options) + opt_info.mutable_offset); + if (!s.ok()) { + return s; } } catch (std::exception& e) { return Status::InvalidArgument("Error parsing " + o.first + ":" + @@ -1027,28 +928,26 @@ Status StringToMap(const std::string& opts_str, } Status GetStringFromStruct( - const ConfigOptions& cfg_options, const void* const opt_ptr, + const ConfigOptions& config_options, const void* const opt_ptr, const std::unordered_map& type_info, std::string* opt_string) { assert(opt_string); opt_string->clear(); for (const auto iter : type_info) { const auto& opt_info = iter.second; - if (opt_info.IsDeprecated()) { - // If the option is no longer used in rocksdb and marked as deprecated, - // we skip it in the serialization. - continue; - } - const char* opt_address = - reinterpret_cast(opt_ptr) + opt_info.offset; - std::string value; - bool result = - SerializeSingleOptionHelper(opt_address, opt_info.type, &value); - if (result) { - opt_string->append(iter.first + "=" + value + cfg_options.delimiter); - } else { - return Status::InvalidArgument("failed to serialize %s\n", - iter.first.c_str()); + // If the option is no longer used in rocksdb and marked as deprecated, + // we skip it in the serialization. + if (opt_info.ShouldSerialize()) { + const char* opt_addr = + reinterpret_cast(opt_ptr) + opt_info.offset; + std::string value; + Status s = opt_info.SerializeOption(config_options, iter.first, opt_addr, + &value); + if (s.ok()) { + opt_string->append(iter.first + "=" + value + config_options.delimiter); + } else { + return s; + } } } return Status::OK(); @@ -1103,41 +1002,14 @@ static Status ParseDBOption(const ConfigOptions& config_options, const std::string& value = config_options.input_strings_escaped ? UnescapeOptionString(org_value) : org_value; - try { - if (name == "rate_limiter_bytes_per_sec") { - new_options->rate_limiter.reset( - NewGenericRateLimiter(static_cast(ParseUint64(value)))); - } else if (name == kNameEnv) { - // Currently `Env` can be deserialized from object registry only. - Env* env = new_options->env; - Status status = Env::LoadEnv(value, &env); - // Only support static env for now. - if (status.ok()) { - new_options->env = env; - } - } else { - auto iter = db_options_type_info.find(name); - if (iter == db_options_type_info.end()) { - return Status::InvalidArgument("Unrecognized option DBOptions:", name); - } - const auto& opt_info = iter->second; - if (opt_info.IsDeprecated() || - ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.offset, - opt_info.type, value)) { - return Status::OK(); - } else if (opt_info.IsByName()) { - return Status::NotSupported("Deserializing the specified DB option " + - name + " is not supported"); - } else { - return Status::InvalidArgument( - "Unable to parse the specified DB option " + name); - } - } - } catch (const std::exception&) { - return Status::InvalidArgument("Unable to parse DBOptions:", name); + auto iter = db_options_type_info.find(name); + if (iter == db_options_type_info.end()) { + return Status::InvalidArgument("Unrecognized option DBOptions:", name); + } else { + return iter->second.ParseOption( + config_options, name, value, + reinterpret_cast(new_options) + iter->second.offset); } - return Status::OK(); } Status GetColumnFamilyOptionsFromMap( @@ -1530,6 +1402,222 @@ std::unordered_map OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; +Status OptionTypeInfo::ParseOption(const ConfigOptions& config_options, + const std::string& opt_name, + const std::string& opt_value, + char* opt_addr) const { + if (IsDeprecated()) { + return Status::OK(); + } + try { + if (opt_addr == nullptr) { + return Status::NotFound("Could not find option: ", opt_name); + } else if (parser_func != nullptr) { + return parser_func(config_options, opt_name, opt_value, opt_addr); + } else if (ParseOptionHelper(opt_addr, type, opt_value)) { + return Status::OK(); + } else if (IsByName()) { + return Status::NotSupported("Deserializing the option " + opt_name + + " is not supported"); + } else { + return Status::InvalidArgument("Error parsing:", opt_name); + } + } catch (std::exception& e) { + return Status::InvalidArgument("Error parsing " + opt_name + ":" + + std::string(e.what())); + } +} + +Status OptionTypeInfo::SerializeOption(const ConfigOptions& config_options, + const std::string& opt_name, + const char* opt_addr, + std::string* opt_value) const { + // If the option is no longer used in rocksdb and marked as deprecated, + // we skip it in the serialization. + Status s; + if (opt_addr == nullptr || IsDeprecated()) { + return Status::OK(); + } else if (string_func != nullptr) { + return string_func(config_options, opt_name, opt_addr, opt_value); + } else if (SerializeSingleOptionHelper(opt_addr, type, opt_value)) { + s = Status::OK(); + } else { + s = Status::InvalidArgument("Cannot serialize option: ", opt_name); + } + return s; +} + +template +bool IsOptionEqual(const char* offset1, const char* offset2) { + return (*reinterpret_cast(offset1) == + *reinterpret_cast(offset2)); +} + +static bool AreEqualDoubles(const double a, const double b) { + return (fabs(a - b) < 0.00001); +} + +static bool AreOptionsEqual(OptionType type, const char* this_offset, + const char* that_offset) { + switch (type) { + case OptionType::kBoolean: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kInt: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kUInt: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kInt32T: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kInt64T: { + int64_t v1, v2; + GetUnaligned(reinterpret_cast(this_offset), &v1); + GetUnaligned(reinterpret_cast(that_offset), &v2); + return (v1 == v2); + } + case OptionType::kVectorInt: + return IsOptionEqual >(this_offset, that_offset); + case OptionType::kUInt32T: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kUInt64T: { + uint64_t v1, v2; + GetUnaligned(reinterpret_cast(this_offset), &v1); + GetUnaligned(reinterpret_cast(that_offset), &v2); + return (v1 == v2); + } + case OptionType::kSizeT: { + size_t v1, v2; + GetUnaligned(reinterpret_cast(this_offset), &v1); + GetUnaligned(reinterpret_cast(that_offset), &v2); + return (v1 == v2); + } + case OptionType::kString: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kDouble: + return AreEqualDoubles(*reinterpret_cast(this_offset), + *reinterpret_cast(that_offset)); + case OptionType::kVectorCompressionType: + return IsOptionEqual >(this_offset, + that_offset); + case OptionType::kCompactionStyle: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kCompactionStopStyle: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kCompactionPri: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kCompressionType: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kChecksumType: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kEncodingType: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kBlockBasedTableIndexType: + return IsOptionEqual(this_offset, + that_offset); + case OptionType::kBlockBasedTableDataBlockIndexType: + return IsOptionEqual( + this_offset, that_offset); + case OptionType::kBlockBasedTableIndexShorteningMode: + return IsOptionEqual( + this_offset, that_offset); + case OptionType::kWALRecoveryMode: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kAccessHint: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kInfoLogLevel: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kCompactionOptionsFIFO: { + CompactionOptionsFIFO lhs = + *reinterpret_cast(this_offset); + CompactionOptionsFIFO rhs = + *reinterpret_cast(that_offset); + if (lhs.max_table_files_size == rhs.max_table_files_size && + lhs.allow_compaction == rhs.allow_compaction) { + return true; + } + return false; + } + case OptionType::kCompactionOptionsUniversal: { + CompactionOptionsUniversal lhs = + *reinterpret_cast(this_offset); + CompactionOptionsUniversal rhs = + *reinterpret_cast(that_offset); + if (lhs.size_ratio == rhs.size_ratio && + lhs.min_merge_width == rhs.min_merge_width && + lhs.max_merge_width == rhs.max_merge_width && + lhs.max_size_amplification_percent == + rhs.max_size_amplification_percent && + lhs.compression_size_percent == rhs.compression_size_percent && + lhs.stop_style == rhs.stop_style && + lhs.allow_trivial_move == rhs.allow_trivial_move) { + return true; + } + return false; + } + default: + return false; + } // End switch +} + +bool OptionTypeInfo::MatchesOption(const ConfigOptions& config_options, + const std::string& opt_name, + const char* this_addr, const char* that_addr, + + std::string* mismatch) const { + if (!config_options.IsCheckEnabled(GetSanityLevel())) { + return true; // If the sanity level is not being checked, skip it + } + if (this_addr == nullptr || that_addr == nullptr) { + if (this_addr == that_addr) { + return true; + } + } else if (equals_func != nullptr) { + if (equals_func(config_options, opt_name, this_addr, that_addr, mismatch)) { + return true; + } + } else if (AreOptionsEqual(type, this_addr, that_addr)) { + return true; + } + if (mismatch->empty()) { + *mismatch = opt_name; + } + return false; +} + +bool OptionTypeInfo::MatchesByName(const ConfigOptions& config_options, + const std::string& opt_name, + const char* this_addr, + const char* that_addr) const { + if (IsByName()) { + std::string that_value; + if (SerializeOption(config_options, opt_name, that_addr, &that_value) + .ok()) { + return MatchesByName(config_options, opt_name, this_addr, that_value); + } + } + return false; +} + +bool OptionTypeInfo::MatchesByName(const ConfigOptions& config_options, + const std::string& opt_name, + const char* opt_addr, + const std::string& that_value) const { + std::string this_value; + if (!IsByName()) { + return false; + } else if (!SerializeOption(config_options, opt_name, opt_addr, &this_value) + .ok()) { + return false; + } else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) { + if (that_value == kNullptrString) { + return true; + } + } else if (IsEnabled(OptionVerificationType::kByNameAllowNull)) { + if (that_value == kNullptrString) { + return true; + } + } + return (this_value == that_value); +} #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/options/options_helper.h b/options/options_helper.h index f5edc1b84..f5450c908 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -57,10 +57,6 @@ Status GetTableFactoryFromMap( std::shared_ptr* table_factory, bool ignore_unknown_options = false); -Status ParseCompressionOptions(const std::string& value, - const std::string& name, - CompressionOptions& compression_opts); - Status GetTableFactoryFromMap( const ConfigOptions& config_options, const std::string& factory_name, const std::unordered_map& opt_map, diff --git a/options/options_parser.cc b/options/options_parser.cc index 50f88db1b..901a8bf9d 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -16,7 +16,6 @@ #include "file/read_write_util.h" #include "file/writable_file_writer.h" #include "options/options_helper.h" -#include "options/options_sanity_check.h" #include "port/port.h" #include "rocksdb/convenience.h" #include "rocksdb/db.h" @@ -518,170 +517,6 @@ std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line, return ""; } -namespace { -bool AreEqualDoubles(const double a, const double b) { - return (fabs(a - b) < 0.00001); -} -} // namespace - -bool AreEqualOptions( - const char* opt1, const char* opt2, const OptionTypeInfo& type_info, - const std::string& opt_name, - const std::unordered_map* opt_map) { - const char* offset1 = opt1 + type_info.offset; - const char* offset2 = opt2 + type_info.offset; - - switch (type_info.type) { - case OptionType::kBoolean: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kInt: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kInt32T: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kInt64T: - { - int64_t v1, v2; - GetUnaligned(reinterpret_cast(offset1), &v1); - GetUnaligned(reinterpret_cast(offset2), &v2); - return (v1 == v2); - } - case OptionType::kVectorInt: - return (*reinterpret_cast*>(offset1) == - *reinterpret_cast*>(offset2)); - case OptionType::kUInt: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kUInt32T: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kUInt64T: - { - uint64_t v1, v2; - GetUnaligned(reinterpret_cast(offset1), &v1); - GetUnaligned(reinterpret_cast(offset2), &v2); - return (v1 == v2); - } - case OptionType::kSizeT: - { - size_t v1, v2; - GetUnaligned(reinterpret_cast(offset1), &v1); - GetUnaligned(reinterpret_cast(offset2), &v2); - return (v1 == v2); - } - case OptionType::kString: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kDouble: - return AreEqualDoubles(*reinterpret_cast(offset1), - *reinterpret_cast(offset2)); - case OptionType::kCompactionStyle: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kCompactionPri: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kCompressionType: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kVectorCompressionType: { - const auto* vec1 = - reinterpret_cast*>(offset1); - const auto* vec2 = - reinterpret_cast*>(offset2); - return (*vec1 == *vec2); - } - case OptionType::kChecksumType: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kBlockBasedTableIndexType: - return ( - *reinterpret_cast( - offset1) == - *reinterpret_cast(offset2)); - case OptionType::kBlockBasedTableDataBlockIndexType: - return ( - *reinterpret_cast( - offset1) == - *reinterpret_cast( - offset2)); - case OptionType::kBlockBasedTableIndexShorteningMode: - return ( - *reinterpret_cast( - offset1) == - *reinterpret_cast( - offset2)); - case OptionType::kWALRecoveryMode: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kAccessHint: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kInfoLogLevel: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); - case OptionType::kCompactionOptionsFIFO: { - CompactionOptionsFIFO lhs = - *reinterpret_cast(offset1); - CompactionOptionsFIFO rhs = - *reinterpret_cast(offset2); - if (lhs.max_table_files_size == rhs.max_table_files_size && - lhs.allow_compaction == rhs.allow_compaction) { - return true; - } - return false; - } - case OptionType::kCompactionOptionsUniversal: { - CompactionOptionsUniversal lhs = - *reinterpret_cast(offset1); - CompactionOptionsUniversal rhs = - *reinterpret_cast(offset2); - if (lhs.size_ratio == rhs.size_ratio && - lhs.min_merge_width == rhs.min_merge_width && - lhs.max_merge_width == rhs.max_merge_width && - lhs.max_size_amplification_percent == - rhs.max_size_amplification_percent && - lhs.compression_size_percent == rhs.compression_size_percent && - lhs.stop_style == rhs.stop_style && - lhs.allow_trivial_move == rhs.allow_trivial_move) { - return true; - } - return false; - } - default: - if (type_info.IsByName()) { - std::string value1; - bool result = - SerializeSingleOptionHelper(offset1, type_info.type, &value1); - if (result == false) { - return false; - } - if (opt_map == nullptr) { - return true; - } - auto iter = opt_map->find(opt_name); - if (iter == opt_map->end()) { - return true; - } else { - if (type_info.IsEnabled(OptionVerificationType::kByNameAllowNull)) { - if (iter->second == kNullptrString || value1 == kNullptrString) { - return true; - } - } else if (type_info.IsEnabled( - OptionVerificationType::kByNameAllowFromNull)) { - if (iter->second == kNullptrString) { - return true; - } - } - return (value1 == iter->second); - } - } - return false; - } -} - Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( const ConfigOptions& config_options, const DBOptions& db_opt, const std::vector& cf_names, @@ -756,34 +591,33 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( Status RocksDBOptionsParser::VerifyDBOptions( const ConfigOptions& config_options, const DBOptions& base_opt, - const DBOptions& persisted_opt, + const DBOptions& file_opt, const std::unordered_map* /*opt_map*/) { for (const auto& pair : db_options_type_info) { - if (pair.second.IsDeprecated()) { - // We skip checking deprecated variables as they might - // contain random values since they might not be initialized - continue; - } - if (DBOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) { - if (!AreEqualOptions(reinterpret_cast(&base_opt), - reinterpret_cast(&persisted_opt), - pair.second, pair.first, nullptr)) { - constexpr size_t kBufferSize = 2048; + const auto& opt_info = pair.second; + if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) { + const char* base_addr = + reinterpret_cast(&base_opt) + opt_info.offset; + const char* file_addr = + reinterpret_cast(&file_opt) + opt_info.offset; + std::string mismatch; + if (!opt_info.MatchesOption(config_options, pair.first, base_addr, + file_addr, &mismatch) && + !opt_info.MatchesByName(config_options, pair.first, base_addr, + file_addr)) { + const size_t kBufferSize = 2048; char buffer[kBufferSize]; std::string base_value; - std::string persisted_value; - SerializeSingleOptionHelper( - reinterpret_cast(&base_opt) + pair.second.offset, - pair.second.type, &base_value); - SerializeSingleOptionHelper( - reinterpret_cast(&persisted_opt) + pair.second.offset, - pair.second.type, &persisted_value); + std::string file_value; + opt_info.SerializeOption(config_options, pair.first, base_addr, + &base_value); + opt_info.SerializeOption(config_options, pair.first, file_addr, + &file_value); snprintf(buffer, sizeof(buffer), "[RocksDBOptionsParser]: " "failed the verification on DBOptions::%s --- " "The specified one is %s while the persisted one is %s.\n", - pair.first.c_str(), base_value.c_str(), - persisted_value.c_str()); + pair.first.c_str(), base_value.c_str(), file_value.c_str()); return Status::InvalidArgument(Slice(buffer, strlen(buffer))); } } @@ -793,38 +627,51 @@ Status RocksDBOptionsParser::VerifyDBOptions( Status RocksDBOptionsParser::VerifyCFOptions( const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, - const ColumnFamilyOptions& persisted_opt, - const std::unordered_map* persisted_opt_map) { + const ColumnFamilyOptions& file_opt, + const std::unordered_map* opt_map) { for (const auto& pair : cf_options_type_info) { - if (pair.second.IsDeprecated()) { - // We skip checking deprecated variables as they might - // contain random values since they might not be initialized - continue; - } - if (CFOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) { - if (!AreEqualOptions(reinterpret_cast(&base_opt), - reinterpret_cast(&persisted_opt), - pair.second, pair.first, persisted_opt_map)) { - constexpr size_t kBufferSize = 2048; + const auto& opt_info = pair.second; + + if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) { + std::string mismatch; + const char* base_addr = + reinterpret_cast(&base_opt) + opt_info.offset; + const char* file_addr = + reinterpret_cast(&file_opt) + opt_info.offset; + bool matches = opt_info.MatchesOption(config_options, pair.first, + base_addr, file_addr, &mismatch); + if (!matches && opt_info.IsByName()) { + if (opt_map == nullptr) { + matches = true; + } else { + auto iter = opt_map->find(pair.first); + if (iter == opt_map->end()) { + matches = true; + } else { + matches = opt_info.MatchesByName(config_options, pair.first, + base_addr, iter->second); + } + } + } + if (!matches) { + // The options do not match + const size_t kBufferSize = 2048; char buffer[kBufferSize]; std::string base_value; - std::string persisted_value; - SerializeSingleOptionHelper( - reinterpret_cast(&base_opt) + pair.second.offset, - pair.second.type, &base_value); - SerializeSingleOptionHelper( - reinterpret_cast(&persisted_opt) + pair.second.offset, - pair.second.type, &persisted_value); + std::string file_value; + opt_info.SerializeOption(config_options, pair.first, base_addr, + &base_value); + opt_info.SerializeOption(config_options, pair.first, file_addr, + &file_value); snprintf(buffer, sizeof(buffer), "[RocksDBOptionsParser]: " "failed the verification on ColumnFamilyOptions::%s --- " "The specified one is %s while the persisted one is %s.\n", - pair.first.c_str(), base_value.c_str(), - persisted_value.c_str()); + pair.first.c_str(), base_value.c_str(), file_value.c_str()); return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); - } - } - } + } // if (! matches) + } // CheckSanityLevel + } // For each option return Status::OK(); } diff --git a/options/options_parser.h b/options/options_parser.h index 2ebdec6d7..ec5d4308a 100644 --- a/options/options_parser.h +++ b/options/options_parser.h @@ -43,11 +43,6 @@ Status PersistRocksDBOptions(const ConfigOptions& config_options, const std::vector& cf_opts, const std::string& file_name, FileSystem* fs); -extern bool AreEqualOptions( - const char* opt1, const char* opt2, const OptionTypeInfo& type_info, - const std::string& opt_name, - const std::unordered_map* opt_map); - class RocksDBOptionsParser { public: explicit RocksDBOptionsParser(); diff --git a/options/options_sanity_check.cc b/options/options_sanity_check.cc deleted file mode 100644 index f73555d6a..000000000 --- a/options/options_sanity_check.cc +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -#ifndef ROCKSDB_LITE - -#include "options/options_sanity_check.h" - -namespace ROCKSDB_NAMESPACE { - -namespace { -ConfigOptions::SanityLevel SanityCheckLevelHelper( - const std::unordered_map& smap, - const std::string& name) { - auto iter = smap.find(name); - return iter != smap.end() ? iter->second - : ConfigOptions::kSanityLevelExactMatch; -} -} - -ConfigOptions::SanityLevel DBOptionSanityCheckLevel( - const std::string& option_name) { - return SanityCheckLevelHelper(sanity_level_db_options, option_name); -} - -ConfigOptions::SanityLevel CFOptionSanityCheckLevel( - const std::string& option_name) { - return SanityCheckLevelHelper(sanity_level_cf_options, option_name); -} - -ConfigOptions::SanityLevel BBTOptionSanityCheckLevel( - const std::string& option_name) { - return SanityCheckLevelHelper(sanity_level_bbt_options, option_name); -} - -} // namespace ROCKSDB_NAMESPACE - -#endif // !ROCKSDB_LITE diff --git a/options/options_sanity_check.h b/options/options_sanity_check.h deleted file mode 100644 index 64f8d9c50..000000000 --- a/options/options_sanity_check.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -#pragma once - -#include -#include - -#include "rocksdb/convenience.h" -#include "rocksdb/rocksdb_namespace.h" - -#ifndef ROCKSDB_LITE -namespace ROCKSDB_NAMESPACE { -// This enum defines the RocksDB options sanity level. - -// The sanity check level for DB options -static const std::unordered_map - sanity_level_db_options{}; - -// The sanity check level for column-family options -static const std::unordered_map - sanity_level_cf_options = { - {"comparator", - ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible}, - {"table_factory", - ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible}, - {"merge_operator", - ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible}}; - -// The sanity check level for block-based table options -static const std::unordered_map - sanity_level_bbt_options{}; - -ConfigOptions::SanityLevel DBOptionSanityCheckLevel( - const std::string& options_name); -ConfigOptions::SanityLevel CFOptionSanityCheckLevel( - const std::string& options_name); -ConfigOptions::SanityLevel BBTOptionSanityCheckLevel( - const std::string& options_name); - -} // namespace ROCKSDB_NAMESPACE - -#endif // !ROCKSDB_LITE diff --git a/options/options_test.cc b/options/options_test.cc index 78f39fd3c..a60794315 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -16,7 +16,6 @@ #include "cache/sharded_cache.h" #include "options/options_helper.h" #include "options/options_parser.h" -#include "options/options_sanity_check.h" #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/convenience.h" @@ -2986,6 +2985,281 @@ TEST_F(OptionsParserTest, EscapeOptionString) { "Escape \\# and # comment together ."), "Escape \\# and"); } + +static void TestAndCompareOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, void* base_ptr, + void* comp_ptr) { + std::string result, mismatch; + char* base_addr = reinterpret_cast(base_ptr) + opt_info.offset; + char* comp_addr = reinterpret_cast(comp_ptr) + opt_info.offset; + ASSERT_OK( + opt_info.SerializeOption(config_options, opt_name, base_addr, &result)); + ASSERT_OK(opt_info.ParseOption(config_options, opt_name, result, comp_addr)); + ASSERT_TRUE(opt_info.MatchesOption(config_options, opt_name, base_addr, + comp_addr, &mismatch)); +} + +template +void TestOptInfo(const ConfigOptions& config_options, OptionType opt_type, + T* base, T* comp) { + std::string result; + OptionTypeInfo opt_info(0, opt_type); + char* base_addr = reinterpret_cast(base); + char* comp_addr = reinterpret_cast(comp); + ASSERT_FALSE(opt_info.MatchesOption(config_options, "base", base_addr, + comp_addr, &result)); + ASSERT_EQ(result, "base"); + ASSERT_NE(*base, *comp); + TestAndCompareOption(config_options, opt_info, "base", base_addr, comp_addr); + ASSERT_EQ(*base, *comp); +} + +class OptionTypeInfoTest : public testing::Test {}; + +TEST_F(OptionTypeInfoTest, BasicTypes) { + ConfigOptions config_options; + { + bool a = true, b = false; + TestOptInfo(config_options, OptionType::kBoolean, &a, &b); + } + { + int a = 100, b = 200; + TestOptInfo(config_options, OptionType::kInt, &a, &b); + } + { + int32_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kInt32T, &a, &b); + } + { + int64_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kInt64T, &a, &b); + } + { + unsigned int a = 100, b = 200; + TestOptInfo(config_options, OptionType::kUInt, &a, &b); + } + { + uint32_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kUInt32T, &a, &b); + } + { + uint64_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kUInt64T, &a, &b); + } + { + size_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kSizeT, &a, &b); + } + { + std::string a = "100", b = "200"; + TestOptInfo(config_options, OptionType::kString, &a, &b); + } + { + double a = 1.0, b = 2.0; + TestOptInfo(config_options, OptionType::kDouble, &a, &b); + } +} + +TEST_F(OptionTypeInfoTest, TestInvalidArgs) { + ConfigOptions config_options; + bool b; + int i; + int32_t i32; + int64_t i64; + unsigned int u; + int32_t u32; + int64_t u64; + size_t sz; + double d; + + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kBoolean) + .ParseOption(config_options, "b", "x", reinterpret_cast(&b))); + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kInt) + .ParseOption(config_options, "b", "x", reinterpret_cast(&i))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt32T) + .ParseOption(config_options, "b", "x", + reinterpret_cast(&i32))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt64T) + .ParseOption(config_options, "b", "x", + reinterpret_cast(&i64))); + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kUInt) + .ParseOption(config_options, "b", "x", reinterpret_cast(&u))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt32T) + .ParseOption(config_options, "b", "x", + reinterpret_cast(&u32))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt64T) + .ParseOption(config_options, "b", "x", + reinterpret_cast(&u64))); + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kSizeT) + .ParseOption(config_options, "b", "x", reinterpret_cast(&sz))); + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kDouble) + .ParseOption(config_options, "b", "x", reinterpret_cast(&d))); + + // Don't know how to convert Unknowns to anything else + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kUnknown) + .ParseOption(config_options, "b", "x", reinterpret_cast(&d))); + + // Verify that if the parse function throws an exception, it is also trapped + OptionTypeInfo func_info(0, OptionType::kUnknown, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0, + [](const ConfigOptions&, const std::string&, + const std::string& value, char* addr) { + auto ptr = reinterpret_cast(addr); + *ptr = ParseInt(value); + return Status::OK(); + }); + ASSERT_OK(func_info.ParseOption(config_options, "b", "1", + reinterpret_cast(&i))); + ASSERT_NOK(func_info.ParseOption(config_options, "b", "x", + reinterpret_cast(&i))); +} + +TEST_F(OptionTypeInfoTest, TestParseFunc) { + OptionTypeInfo opt_info( + 0, OptionType::kUnknown, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0, + [](const ConfigOptions& /*opts*/, const std::string& name, + const std::string& value, char* addr) { + auto ptr = reinterpret_cast(addr); + if (name == "Oops") { + return Status::InvalidArgument(value); + } else { + *ptr = value + " " + name; + return Status::OK(); + } + }); + ConfigOptions config_options; + std::string base; + ASSERT_OK(opt_info.ParseOption(config_options, "World", "Hello", + reinterpret_cast(&base))); + ASSERT_EQ(base, "Hello World"); + ASSERT_NOK(opt_info.ParseOption(config_options, "Oops", "Hello", + reinterpret_cast(&base))); +} + +TEST_F(OptionTypeInfoTest, TestSerializeFunc) { + OptionTypeInfo opt_info( + 0, OptionType::kString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0, nullptr, + [](const ConfigOptions& /*opts*/, const std::string& name, + const char* /*addr*/, std::string* value) { + if (name == "Oops") { + return Status::InvalidArgument(name); + } else { + *value = name; + return Status::OK(); + } + }, + nullptr); + ConfigOptions config_options; + std::string base; + std::string value; + ASSERT_OK(opt_info.SerializeOption(config_options, "Hello", + reinterpret_cast(&base), &value)); + ASSERT_EQ(value, "Hello"); + ASSERT_NOK(opt_info.SerializeOption(config_options, "Oops", + reinterpret_cast(&base), &value)); +} + +TEST_F(OptionTypeInfoTest, TestEqualsFunc) { + OptionTypeInfo opt_info( + 0, OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0, nullptr, nullptr, + [](const ConfigOptions& /*opts*/, const std::string& name, + const char* addr1, const char* addr2, std::string* mismatch) { + auto i1 = *(reinterpret_cast(addr1)); + auto i2 = *(reinterpret_cast(addr2)); + if (name == "LT") { + return i1 < i2; + } else if (name == "GT") { + return i1 > i2; + } else if (name == "EQ") { + return i1 == i2; + } else { + *mismatch = name + "???"; + return false; + } + }); + + ConfigOptions config_options; + int int1 = 100; + int int2 = 200; + std::string mismatch; + ASSERT_TRUE(opt_info.MatchesOption( + config_options, "LT", reinterpret_cast(&int1), + reinterpret_cast(&int2), &mismatch)); + ASSERT_EQ(mismatch, ""); + ASSERT_FALSE(opt_info.MatchesOption( + config_options, "GT", reinterpret_cast(&int1), + reinterpret_cast(&int2), &mismatch)); + ASSERT_EQ(mismatch, "GT"); + ASSERT_FALSE(opt_info.MatchesOption( + config_options, "NO", reinterpret_cast(&int1), + reinterpret_cast(&int2), &mismatch)); + ASSERT_EQ(mismatch, "NO???"); +} + +TEST_F(OptionTypeInfoTest, TestOptionFlags) { + OptionTypeInfo opt_none(0, OptionType::kString, + OptionVerificationType::kNormal, + OptionTypeFlags::kDontSerialize, 0); + OptionTypeInfo opt_never(0, OptionType::kString, + OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever, 0); + OptionTypeInfo opt_alias(0, OptionType::kString, + OptionVerificationType::kAlias, + OptionTypeFlags::kNone, 0); + OptionTypeInfo opt_deprecated(0, OptionType::kString, + OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0); + ConfigOptions config_options; + std::string base = "base"; + std::string comp = "comp"; + + // If marked string none, the serialization returns okay but does nothing + ASSERT_OK(opt_none.SerializeOption(config_options, "None", + reinterpret_cast(&base), &base)); + // If marked never compare, they match even when they do not + ASSERT_TRUE(opt_never.MatchesOption(config_options, "Never", + reinterpret_cast(&base), + reinterpret_cast(&comp), &base)); + ASSERT_FALSE(opt_none.MatchesOption(config_options, "Never", + reinterpret_cast(&base), + reinterpret_cast(&comp), &base)); + + // An alias can change the value via parse, but does nothing on serialize on + // match + std::string result; + ASSERT_OK(opt_alias.ParseOption(config_options, "Alias", "Alias", + reinterpret_cast(&base))); + ASSERT_OK(opt_alias.SerializeOption(config_options, "Alias", + reinterpret_cast(&base), &result)); + ASSERT_TRUE(opt_alias.MatchesOption(config_options, "Alias", + reinterpret_cast(&base), + reinterpret_cast(&comp), &result)); + ASSERT_EQ(base, "Alias"); + ASSERT_NE(base, comp); + + // Deprecated options do nothing on any of the commands + ASSERT_OK(opt_deprecated.ParseOption(config_options, "Alias", "Deprecated", + reinterpret_cast(&base))); + ASSERT_OK(opt_deprecated.SerializeOption( + config_options, "Alias", reinterpret_cast(&base), &result)); + ASSERT_TRUE(opt_deprecated.MatchesOption( + config_options, "Alias", reinterpret_cast(&base), + reinterpret_cast(&comp), &result)); + ASSERT_EQ(base, "Alias"); + ASSERT_NE(base, comp); +} + #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/options/options_type.h b/options/options_type.h index 51e0c0031..b1769a712 100644 --- a/options/options_type.h +++ b/options/options_type.h @@ -5,7 +5,13 @@ #pragma once +#include +#include +#include + +#include "rocksdb/convenience.h" #include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" namespace ROCKSDB_NAMESPACE { @@ -58,16 +64,25 @@ enum class OptionVerificationType { // where one of them is a nullptr. kByNameAllowFromNull, // Same as kByName, but it also allows the case // where the old option is nullptr. - kDeprecated // The option is no longer used in rocksdb. The RocksDB + kDeprecated, // The option is no longer used in rocksdb. The RocksDB // OptionsParser will still accept this option if it // happen to exists in some Options file. However, // the parser will not include it in serialization // and verification processes. + kAlias, // This option represents is a name/shortcut for + // another option and should not be written or verified + // independently }; enum class OptionTypeFlags : uint32_t { - kNone = 0x00, // No flags - kMutable = 0x01, // Option is mutable + kNone = 0x00, // No flags + kCompareDefault = 0x0, + kCompareNever = ConfigOptions::kSanityLevelNone, + kCompareLoose = ConfigOptions::kSanityLevelLooselyCompatible, + kCompareExact = ConfigOptions::kSanityLevelExactMatch, + + kMutable = 0x0100, // Option is mutable + kDontSerialize = 0x2000, // Don't serialize the option }; inline OptionTypeFlags operator|(const OptionTypeFlags &a, @@ -82,18 +97,55 @@ inline OptionTypeFlags operator&(const OptionTypeFlags &a, static_cast(b)); } +// Function for converting a option string value into its underlying +// representation in "addr" +// On success, Status::OK is returned and addr is set to the parsed form +// On failure, a non-OK status is returned +// @param opts The ConfigOptions controlling how the value is parsed +// @param name The name of the options being parsed +// @param value The string representation of the option +// @param addr Pointer to the object +using ParserFunc = std::function; + +// Function for converting an option "addr" into its string representation. +// On success, Status::OK is returned and value is the serialized form. +// On failure, a non-OK status is returned +// @param opts The ConfigOptions controlling how the values are serialized +// @param name The name of the options being serialized +// @param addr Pointer to the value being serialized +// @param value The result of the serialization. +using StringFunc = std::function; + +// Function for comparing two option values +// If they are not equal, updates "mismatch" with the name of the bad option +// @param opts The ConfigOptions controlling how the values are compared +// @param name The name of the options being compared +// @param addr1 The first address to compare +// @param addr2 The address to compare to +// @param mismatch If the values are not equal, the name of the option that +// first differs +using EqualsFunc = std::function; + // A struct for storing constant option information such as option name, // option type, and offset. class OptionTypeInfo { public: int offset; int mutable_offset; - OptionType type; // A simple "normal", non-mutable Type "_type" at _offset OptionTypeInfo(int _offset, OptionType _type) : offset(_offset), mutable_offset(0), + parser_func(nullptr), + string_func(nullptr), + equals_func(nullptr), type(_type), verification(OptionVerificationType::kNormal), flags(OptionTypeFlags::kNone) {} @@ -102,6 +154,9 @@ class OptionTypeInfo { OptionTypeInfo(int _offset, OptionType _type, int _mutable_offset) : offset(_offset), mutable_offset(_mutable_offset), + parser_func(nullptr), + string_func(nullptr), + equals_func(nullptr), type(_type), verification(OptionVerificationType::kNormal), flags(OptionTypeFlags::kMutable) {} @@ -111,6 +166,34 @@ class OptionTypeInfo { int _mutable_offset) : offset(_offset), mutable_offset(_mutable_offset), + parser_func(nullptr), + string_func(nullptr), + equals_func(nullptr), + type(_type), + verification(_verification), + flags(_flags) {} + + OptionTypeInfo(int _offset, OptionType _type, + OptionVerificationType _verification, OptionTypeFlags _flags, + int _mutable_offset, const ParserFunc& _pfunc) + : offset(_offset), + mutable_offset(_mutable_offset), + parser_func(_pfunc), + string_func(nullptr), + equals_func(nullptr), + type(_type), + verification(_verification), + flags(_flags) {} + + OptionTypeInfo(int _offset, OptionType _type, + OptionVerificationType _verification, OptionTypeFlags _flags, + int _mutable_offset, const ParserFunc& _pfunc, + const StringFunc& _sfunc, const EqualsFunc& _efunc) + : offset(_offset), + mutable_offset(_mutable_offset), + parser_func(_pfunc), + string_func(_sfunc), + equals_func(_efunc), type(_type), verification(_verification), flags(_flags) {} @@ -123,18 +206,93 @@ class OptionTypeInfo { return IsEnabled(OptionVerificationType::kDeprecated); } + // Returns true if the option is marked as an Alias. + // Aliases are valid options that are parsed but are not converted to strings + // or compared. + bool IsAlias() const { return IsEnabled(OptionVerificationType::kAlias); } + bool IsEnabled(OptionVerificationType ovf) const { return verification == ovf; } + // Returns the sanity level for comparing the option. + // If the options should not be compared, returns None + // If the option has a compare flag, returns it. + // Otherwise, returns "exact" + ConfigOptions::SanityLevel GetSanityLevel() const { + if (IsDeprecated() || IsAlias()) { + return ConfigOptions::SanityLevel::kSanityLevelNone; + } else { + auto match = (flags & OptionTypeFlags::kCompareExact); + if (match == OptionTypeFlags::kCompareDefault) { + return ConfigOptions::SanityLevel::kSanityLevelExactMatch; + } else { + return (ConfigOptions::SanityLevel)match; + } + } + } + + // Returns true if the option should be serialized. + // Options should be serialized if the are not deprecated, aliases, + // or marked as "Don't Serialize". + bool ShouldSerialize() const { + if (IsDeprecated() || IsAlias()) { + return false; + } else if (IsEnabled(OptionTypeFlags::kDontSerialize)) { + return false; + } else { + return true; + } + } + bool IsByName() const { return (verification == OptionVerificationType::kByName || verification == OptionVerificationType::kByNameAllowNull || verification == OptionVerificationType::kByNameAllowFromNull); } - protected: + // Parses the option in "opt_value" according to the rules of this class + // and updates the value at "opt_addr". + // On success, Status::OK() is returned. On failure: + // NotFound means the opt_name is not valid for this option + // NotSupported means we do not know how to parse the value for this option + // InvalidArgument means the opt_value is not valid for this option. + Status ParseOption(const ConfigOptions& config_options, + const std::string& opt_name, const std::string& opt_value, + char* opt_addr) const; + + // Serializes the option in "opt_addr" according to the rules of this class + // into the value at "opt_value". + Status SerializeOption(const ConfigOptions& config_options, + const std::string& opt_name, const char* opt_addr, + std::string* opt_value) const; + + // Compares the "addr1" and "addr2" values according to the rules of this + // class and returns true if they match. On a failed match, mismatch is the + // name of the option that failed to match. + bool MatchesOption(const ConfigOptions& config_options, + const std::string& opt_name, const char* addr1, + const char* addr2, std::string* mismatch) const; + + // Used to override the match rules for "ByName" options. + bool MatchesByName(const ConfigOptions& config_options, + const std::string& opt_name, const char* this_offset, + const char* that_offset) const; + bool MatchesByName(const ConfigOptions& config_options, + const std::string& opt_name, const char* this_ptr, + const std::string& that_value) const; + private: + // The optional function to convert a string to its representation + ParserFunc parser_func; + + // The optional function to convert a value to its string representation + StringFunc string_func; + + // The optional function to convert a match to option values + EqualsFunc equals_func; + + OptionType type; OptionVerificationType verification; OptionTypeFlags flags; }; diff --git a/src.mk b/src.mk index 5fc865607..1c28f3e0e 100644 --- a/src.mk +++ b/src.mk @@ -1,5 +1,6 @@ # These are the sources from which librocksdb.a is built: LIB_SOURCES = \ + cache/cache.cc \ cache/clock_cache.cc \ cache/lru_cache.cc \ cache/sharded_cache.cc \ @@ -118,7 +119,6 @@ LIB_SOURCES = \ options/options.cc \ options/options_helper.cc \ options/options_parser.cc \ - options/options_sanity_check.cc \ port/port_posix.cc \ port/stack_trace.cc \ table/adaptive/adaptive_table_factory.cc \ diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index d2d507b3e..ed3e4c3cd 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -17,7 +17,6 @@ #include "options/options_helper.h" #include "options/options_parser.h" -#include "options/options_sanity_check.h" #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/convenience.h" @@ -170,7 +169,7 @@ static std::unordered_map {"flush_block_policy_factory", {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kCompareNever, 0}}, {"cache_index_and_filter_blocks", {offsetof(struct BlockBasedTableOptions, cache_index_and_filter_blocks), @@ -244,8 +243,46 @@ static std::unordered_map OptionTypeFlags::kNone, 0}}, {"filter_policy", {offsetof(struct BlockBasedTableOptions, filter_policy), - OptionType::kFilterPolicy, OptionVerificationType::kByName, - OptionTypeFlags::kNone, 0}}, + OptionType::kUnknown, OptionVerificationType::kByNameAllowFromNull, + OptionTypeFlags::kNone, 0, + // Parses the Filter policy + [](const ConfigOptions& opts, const std::string&, + const std::string& value, char* addr) { + auto* policy = + reinterpret_cast*>(addr); + return FilterPolicy::CreateFromString(opts, value, policy); + }, + // Converts the FilterPolicy to its string representation + [](const ConfigOptions&, const std::string&, const char* addr, + std::string* value) { + const auto* policy = + reinterpret_cast*>( + addr); + if (policy->get()) { + *value = (*policy)->Name(); + } else { + *value = kNullptrString; + } + return Status::OK(); + }, + // Compares two FilterPolicy objects for equality + [](const ConfigOptions&, const std::string&, const char* addr1, + const char* addr2, std::string*) { + const auto* policy1 = + reinterpret_cast*>( + addr1) + ->get(); + const auto* policy2 = + reinterpret_cast*>(addr2) + ->get(); + if (policy1 == policy2) { + return true; + } else if (policy1 != nullptr && policy2 != nullptr) { + return (strcmp(policy1->Name(), policy2->Name()) == 0); + } else { + return false; + } + }}}, {"whole_key_filtering", {offsetof(struct BlockBasedTableOptions, whole_key_filtering), OptionType::kBoolean, OptionVerificationType::kNormal, @@ -277,7 +314,28 @@ static std::unordered_map {offsetof(struct BlockBasedTableOptions, pin_top_level_index_and_filter), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}}; + OptionTypeFlags::kNone, 0}}, + {"block_cache", + {offsetof(struct BlockBasedTableOptions, block_cache), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), 0, + // Parses the input vsalue as a Cache + [](const ConfigOptions& opts, const std::string&, + const std::string& value, char* addr) { + auto* cache = reinterpret_cast*>(addr); + return Cache::CreateFromString(opts, value, cache); + }}}, + {"block_cache_compressed", + {offsetof(struct BlockBasedTableOptions, block_cache_compressed), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), 0, + // Parses the input vsalue as a Cache + [](const ConfigOptions& opts, const std::string&, + const std::string& value, char* addr) { + auto* cache = reinterpret_cast*>(addr); + return Cache::CreateFromString(opts, value, cache); + }}}, +}; #endif // ROCKSDB_LITE // TODO(myabandeh): We should return an error instead of silently changing the @@ -566,55 +624,6 @@ std::string ParseBlockBasedTableOption(const ConfigOptions& config_options, const std::string& value = config_options.input_strings_escaped ? UnescapeOptionString(org_value) : org_value; - if (!config_options.input_strings_escaped) { - // if the input string is not escaped, it means this function is - // invoked from SetOptions, which takes the old format. - if (name == "block_cache" || name == "block_cache_compressed") { - // cache options can be specified in the following format - // "block_cache={capacity=1M;num_shard_bits=4; - // strict_capacity_limit=true;high_pri_pool_ratio=0.5;}" - // To support backward compatibility, the following format - // is also supported. - // "block_cache=1M" - std::shared_ptr cache; - // block_cache is specified in format block_cache=. - if (value.find('=') == std::string::npos) { - cache = NewLRUCache(ParseSizeT(value)); - } else { - LRUCacheOptions cache_opts; - if (!ParseOptionHelper(reinterpret_cast(&cache_opts), - OptionType::kLRUCacheOptions, value)) { - return "Invalid cache options"; - } - cache = NewLRUCache(cache_opts); - } - - if (name == "block_cache") { - new_options->block_cache = cache; - } else { - new_options->block_cache_compressed = cache; - } - return ""; - } else if (name == "filter_policy") { - // Expect the following format - // bloomfilter:int:bool - const std::string kName = "bloomfilter:"; - if (value.compare(0, kName.size(), kName) != 0) { - return "Invalid filter policy name"; - } - size_t pos = value.find(':', kName.size()); - if (pos == std::string::npos) { - return "Invalid filter policy config, missing bits_per_key"; - } - double bits_per_key = - ParseDouble(trim(value.substr(kName.size(), pos - kName.size()))); - bool use_block_based_builder = - ParseBoolean("use_block_based_builder", trim(value.substr(pos + 1))); - new_options->filter_policy.reset( - NewBloomFilterPolicy(bits_per_key, use_block_based_builder)); - return ""; - } - } const auto iter = block_based_table_type_info.find(name); if (iter == block_based_table_type_info.end()) { if (config_options.ignore_unknown_options) { @@ -624,12 +633,14 @@ std::string ParseBlockBasedTableOption(const ConfigOptions& config_options, } } const auto& opt_info = iter->second; - if (!opt_info.IsDeprecated() && - !ParseOptionHelper(reinterpret_cast(new_options) + opt_info.offset, - opt_info.type, value)) { - return "Invalid value"; + Status s = opt_info.ParseOption( + config_options, iter->first, value, + reinterpret_cast(new_options) + opt_info.offset); + if (s.ok()) { + return ""; + } else { + return s.ToString(); } - return ""; } } // namespace @@ -712,16 +723,20 @@ Status VerifyBlockBasedTableFactory(const ConfigOptions& config_options, const auto& base_opt = base_tf->table_options(); const auto& file_opt = file_tf->table_options(); + std::string mismatch; for (auto& pair : block_based_table_type_info) { - if (pair.second.IsDeprecated()) { - // We skip checking deprecated variables as they might - // contain random values since they might not be initialized - continue; - } - if (BBTOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) { - if (!AreEqualOptions(reinterpret_cast(&base_opt), - reinterpret_cast(&file_opt), - pair.second, pair.first, nullptr)) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) { + const char* base_addr = + reinterpret_cast(&base_opt) + pair.second.offset; + const char* file_addr = + reinterpret_cast(&file_opt) + pair.second.offset; + + if (!pair.second.MatchesOption(config_options, pair.first, base_addr, + file_addr, &mismatch) && + !pair.second.MatchesByName(config_options, pair.first, base_addr, + file_addr)) { return Status::Corruption( "[RocksDBOptionsParser]: " "failed the verification on BlockBasedTableOptions::", diff --git a/table/block_based/filter_policy.cc b/table/block_based/filter_policy.cc index c8f23ee33..5fc63fd88 100644 --- a/table/block_based/filter_policy.cc +++ b/table/block_based/filter_policy.cc @@ -756,4 +756,34 @@ FilterBuildingContext::FilterBuildingContext( FilterPolicy::~FilterPolicy() { } +Status FilterPolicy::CreateFromString( + const ConfigOptions& /*options*/, const std::string& value, + std::shared_ptr* policy) { + const std::string kBloomName = "bloomfilter:"; + if (value == kNullptrString || value == "rocksdb.BuiltinBloomFilter") { + policy->reset(); +#ifndef ROCKSDB_LITE + } else if (value.compare(0, kBloomName.size(), kBloomName) == 0) { + size_t pos = value.find(':', kBloomName.size()); + if (pos == std::string::npos) { + return Status::InvalidArgument( + "Invalid filter policy config, missing bits_per_key"); + } else { + double bits_per_key = ParseDouble( + trim(value.substr(kBloomName.size(), pos - kBloomName.size()))); + bool use_block_based_builder = + ParseBoolean("use_block_based_builder", trim(value.substr(pos + 1))); + policy->reset( + NewBloomFilterPolicy(bits_per_key, use_block_based_builder)); + } + } else { + return Status::InvalidArgument("Invalid filter policy name ", value); +#else + } else { + return Status::NotSupported("Cannot load filter policy in LITE mode ", + value); +#endif // ROCKSDB_LITE + } + return Status::OK(); +} } // namespace ROCKSDB_NAMESPACE diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index 4759eb975..8b498a6da 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -220,12 +220,14 @@ std::string ParsePlainTableOptions(const ConfigOptions& config_options, } } const auto& opt_info = iter->second; - if (!opt_info.IsDeprecated() && - !ParseOptionHelper(reinterpret_cast(new_options) + opt_info.offset, - opt_info.type, value)) { - return "Invalid value"; + Status s = opt_info.ParseOption( + config_options, name, value, + reinterpret_cast(new_options) + opt_info.offset); + if (s.ok()) { + return ""; + } else { + return s.ToString(); } - return ""; } Status GetPlainTableOptionsFromMap(