From 4cbc19d2a1da1a44775f27c2d167a0d8da0fd9f5 Mon Sep 17 00:00:00 2001 From: mrambacher Date: Tue, 21 Apr 2020 17:35:28 -0700 Subject: [PATCH] Add a ConfigOptions for use in comparing objects and converting to/from strings (#6389) Summary: The methods in convenience.h are used to compare/convert objects to/from strings. There is a mishmash of parameters in use here with more needed in the future. This PR replaces those parameters with a single structure. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6389 Reviewed By: siying Differential Revision: D21163707 Pulled By: zhichao-cao fbshipit-source-id: f807b4cc7e2b0af3871536b69546b2604dfa81bd --- HISTORY.md | 1 + db/column_family_test.cc | 8 +- db/db_bloom_filter_test.cc | 1 + db/db_options_test.cc | 9 +- examples/options_file_example.cc | 3 +- include/rocksdb/convenience.h | 149 +- include/rocksdb/table.h | 10 +- include/rocksdb/utilities/ldb_cmd.h | 5 +- include/rocksdb/utilities/options_util.h | 28 +- java/CMakeLists.txt | 3 + java/Makefile | 1 + java/rocksjni/config_options.cc | 88 + java/rocksjni/options.cc | 71 +- java/rocksjni/options_util.cc | 64 +- java/rocksjni/portal.h | 37 + .../java/org/rocksdb/ColumnFamilyOptions.java | 47 +- .../main/java/org/rocksdb/ConfigOptions.java | 47 + java/src/main/java/org/rocksdb/DBOptions.java | 51 +- java/src/main/java/org/rocksdb/Options.java | 24 +- .../main/java/org/rocksdb/OptionsUtil.java | 45 +- .../main/java/org/rocksdb/SanityLevel.java | 42 + .../org/rocksdb/ColumnFamilyOptionsTest.java | 21 + options/cf_options.cc | 11 +- options/options_helper.cc | 196 +- options/options_helper.h | 37 +- options/options_parser.cc | 140 +- options/options_parser.h | 51 +- options/options_sanity_check.cc | 13 +- options/options_sanity_check.h | 35 +- options/options_test.cc | 1786 +++++++++++++---- src.mk | 1 + .../block_based/block_based_table_factory.cc | 81 +- table/block_based/block_based_table_factory.h | 14 +- table/block_based/block_based_table_reader.cc | 6 +- table/cuckoo/cuckoo_table_factory.h | 4 +- table/plain/plain_table_factory.cc | 54 +- table/plain/plain_table_factory.h | 4 +- tools/ldb_cmd.cc | 9 +- utilities/options/options_util.cc | 63 +- utilities/options/options_util_test.cc | 75 +- 40 files changed, 2582 insertions(+), 753 deletions(-) create mode 100644 java/rocksjni/config_options.cc create mode 100644 java/src/main/java/org/rocksdb/ConfigOptions.java create mode 100644 java/src/main/java/org/rocksdb/SanityLevel.java diff --git a/HISTORY.md b/HISTORY.md index 6e9a6578d..893d46040 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,7 @@ * Finish implementation of BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey. It's now ready for use. Significantly reduces read amplification in some setups, especially for iterator seeks. ### Public API Change +* Add a ConfigOptions argument to the APIs dealing with converting options to and from strings and files. The ConfigOptions is meant to replace some of the options (such as input_strings_escaped and ignore_unknown_options) and allow for more parameters to be passed in the future without changing the function signature. * Add NewFileChecksumGenCrc32cFactory to the file checksum public API, such that the builtin Crc32c based file checksum generator factory can be used by applications. * Add IsDirectory to Env and FS to indicate if a path is a directory. diff --git a/db/column_family_test.cc b/db/column_family_test.cc index feee3dcf4..fa791506f 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -8,16 +8,16 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include -#include #include #include +#include #include "db/db_impl/db_impl.h" #include "db/db_test_util.h" -#include "memtable/hash_skiplist_rep.h" #include "options/options_parser.h" #include "port/port.h" #include "port/stack_trace.h" +#include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/iterator.h" @@ -287,7 +287,9 @@ class ColumnFamilyTestBase : public testing::Test { // Verify the CF options of the returned CF handle. ColumnFamilyDescriptor desc; ASSERT_OK(handles_[cfi]->GetDescriptor(&desc)); - RocksDBOptionsParser::VerifyCFOptions(desc.options, current_cf_opt); + RocksDBOptionsParser::VerifyCFOptions(ConfigOptions(), desc.options, + current_cf_opt); + #endif // !ROCKSDB_LITE cfi++; } diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index 00f2f07eb..194c87732 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -8,6 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/db_test_util.h" +#include "options/options_helper.h" #include "port/stack_trace.h" #include "rocksdb/perf_context.h" #include "table/block_based/filter_policy_internal.h" diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 76ee27224..a884ac58e 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -33,7 +33,9 @@ class DBOptionsTest : public DBTestBase { std::unordered_map GetMutableDBOptionsMap( const DBOptions& options) { std::string options_str; - GetStringFromDBOptions(&options_str, options); + ConfigOptions config_options; + config_options.delimiter = "; "; + GetStringFromDBOptions(config_options, options, &options_str); std::unordered_map options_map; StringToMap(options_str, &options_map); std::unordered_map mutable_map; @@ -48,7 +50,10 @@ class DBOptionsTest : public DBTestBase { std::unordered_map GetMutableCFOptionsMap( const ColumnFamilyOptions& options) { std::string options_str; - GetStringFromColumnFamilyOptions(&options_str, options); + ConfigOptions config_options; + config_options.delimiter = "; "; + + GetStringFromColumnFamilyOptions(config_options, options, &options_str); std::unordered_map options_map; StringToMap(options_str, &options_map); std::unordered_map mutable_map; diff --git a/examples/options_file_example.cc b/examples/options_file_example.cc index e6a1d0e9a..30051d8d5 100644 --- a/examples/options_file_example.cc +++ b/examples/options_file_example.cc @@ -79,7 +79,8 @@ int main() { // Load the options file. DBOptions loaded_db_opt; std::vector loaded_cf_descs; - s = LoadLatestOptions(kDBPath, Env::Default(), &loaded_db_opt, + ConfigOptions config_options; + s = LoadLatestOptions(config_options, kDBPath, &loaded_db_opt, &loaded_cf_descs); assert(s.ok()); assert(loaded_db_opt.create_if_missing == db_opt.create_if_missing); diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index 442303d94..d8843cb76 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -10,12 +10,74 @@ #include #include "rocksdb/db.h" -#include "rocksdb/options.h" +#include "rocksdb/status.h" #include "rocksdb/table.h" namespace ROCKSDB_NAMESPACE { +class Env; +struct ColumnFamilyOptions; +struct DBOptions; +struct Options; + +// ConfigOptions containing the parameters/controls for +// comparing objects and converting to/from strings. +// These settings control how the methods +// treat errors (e.g. ignore_unknown_objects), the format +// of the serialization (e.g. delimiter), and how to compare +// options (sanity_level). +struct ConfigOptions { + // This enum defines the RocksDB options sanity level. + enum SanityLevel : unsigned char { + kSanityLevelNone = 0x01, // Performs no sanity check at all. + // Performs minimum check to ensure the RocksDB instance can be + // opened without corrupting / mis-interpreting the data. + kSanityLevelLooselyCompatible = 0x02, + // Perform exact match sanity check. + kSanityLevelExactMatch = 0xFF, + }; + + enum Depth { + kDepthDefault, // Traverse nested options that are not flagged as "shallow" + kDepthShallow, // Do not traverse into any nested options + kDepthDetailed, // Traverse nested options, overriding the options shallow + // setting + }; + + // When true, any unused options will be ignored and OK will be returned + bool ignore_unknown_options = false; + + // If the strings are escaped (old-style?) + bool input_strings_escaped = true; + + // The separator between options when converting to a string + std::string delimiter = ";"; + + // Controls how to traverse options during print/match stages + Depth depth = Depth::kDepthDefault; + + // Controls how options are serialized + // Controls how pedantic the comparison must be for equivalency + SanityLevel sanity_level = SanityLevel::kSanityLevelExactMatch; + // `file_readahead_size` is used for readahead for the option file. + size_t file_readahead_size = 512 * 1024; + + // The environment to use for this option + Env* env = Env::Default(); + + bool IsShallow() const { return depth == Depth::kDepthShallow; } + bool IsDetailed() const { return depth == Depth::kDepthDetailed; } + + bool IsCheckDisabled() const { + return sanity_level == SanityLevel::kSanityLevelNone; + } + + bool IsCheckEnabled(SanityLevel level) const { + return (level > SanityLevel::kSanityLevelNone && level <= sanity_level); + } +}; #ifndef ROCKSDB_LITE + // The following set of functions provide a way to construct RocksDB Options // from a string or a string-to-string map. Here're the general rule of // setting option values from strings by type. Some RocksDB types are also @@ -134,13 +196,6 @@ namespace ROCKSDB_NAMESPACE { // [Example]: // * {"memtable", "vector:1024"} is equivalent to setting memtable // to VectorRepFactory(1024). -// - HashCuckooRepFactory: -// Pass "cuckoo:" to use HashCuckooRepFactory with the -// specified write buffer size, or simply "cuckoo" to use the default -// HashCuckooRepFactory. -// [Example]: -// * {"memtable", "cuckoo:1024"} is equivalent to setting memtable -// to NewHashCuckooRepFactory(1024). // // * compression_opts: // Use "compression_opts" to config compression_opts. The value format @@ -153,6 +208,12 @@ namespace ROCKSDB_NAMESPACE { // cf_opt.compression_opts.strategy = 6; // cf_opt.compression_opts.max_dict_bytes = 7; // +// The GetColumnFamilyOptionsFromMap(ConfigOptions, ...) should be used; the +// alternative signature may be deprecated in a future release. The equivalent +// functionality can be achieved by setting the corresponding options in +// the ConfigOptions parameter. +// +// @param config_options controls how the map is processed. // @param base_options the default options of the output "new_options". // @param opts_map an option name to value map for specifying how "new_options" // should be set. @@ -165,6 +226,11 @@ namespace ROCKSDB_NAMESPACE { // instead of resulting in an unknown-option error. // @return Status::OK() on success. Otherwise, a non-ok status indicating // error will be returned, and "new_options" will be set to "base_options". +Status GetColumnFamilyOptionsFromMap( + const ConfigOptions& config_options, + const ColumnFamilyOptions& base_options, + const std::unordered_map& opts_map, + ColumnFamilyOptions* new_options); Status GetColumnFamilyOptionsFromMap( const ColumnFamilyOptions& base_options, const std::unordered_map& opts_map, @@ -184,6 +250,12 @@ Status GetColumnFamilyOptionsFromMap( // - Passing {"rate_limiter_bytes_per_sec", "1024"} is equivalent to // passing NewGenericRateLimiter(1024) to rate_limiter_bytes_per_sec. // +// The GetDBOptionsFromMap(ConfigOptions, ...) should be used; the +// alternative signature may be deprecated in a future release. The equivalent +// functionality can be achieved by setting the corresponding options in +// the ConfigOptions parameter. +// +// @param config_options controls how the map is processed. // @param base_options the default options of the output "new_options". // @param opts_map an option name to value map for specifying how "new_options" // should be set. @@ -196,6 +268,10 @@ Status GetColumnFamilyOptionsFromMap( // instead of resulting in an unknown-option error. // @return Status::OK() on success. Otherwise, a non-ok status indicating // error will be returned, and "new_options" will be set to "base_options". +Status GetDBOptionsFromMap( + const ConfigOptions& cfg_options, const DBOptions& base_options, + const std::unordered_map& opts_map, + DBOptions* new_options); Status GetDBOptionsFromMap( const DBOptions& base_options, const std::unordered_map& opts_map, @@ -227,6 +303,12 @@ Status GetDBOptionsFromMap( // - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is // equivalent to setting block_cache using NewLRUCache(1024 * 1024). // +// The GetBlockBasedTableOptionsFromMap(ConfigOptions, ...) should be used; +// the alternative signature may be deprecated in a future release. The +// equivalent functionality can be achieved by setting the corresponding +// options in the ConfigOptions parameter. +// +// @param config_options controls how the map is processed. // @param table_options the default options of the output "new_table_options". // @param opts_map an option name to value map for specifying how // "new_table_options" should be set. @@ -240,6 +322,11 @@ Status GetDBOptionsFromMap( // @return Status::OK() on success. Otherwise, a non-ok status indicating // error will be returned, and "new_table_options" will be set to // "table_options". +Status GetBlockBasedTableOptionsFromMap( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, + const std::unordered_map& opts_map, + BlockBasedTableOptions* new_table_options); Status GetBlockBasedTableOptionsFromMap( const BlockBasedTableOptions& table_options, const std::unordered_map& opts_map, @@ -250,6 +337,12 @@ Status GetBlockBasedTableOptionsFromMap( // map "opts_map" of option name to option value to construct the new // PlainTableOptions "new_table_options". // +// The GetPlainTableOptionsFromMap(ConfigOptions, ...) should be used; the +// alternative signature may be deprecated in a future release. The equivalent +// functionality can be achieved by setting the corresponding options in +// the ConfigOptions parameter. +// +// @param config_options controls how the map is processed. // @param table_options the default options of the output "new_table_options". // @param opts_map an option name to value map for specifying how // "new_table_options" should be set. @@ -263,13 +356,17 @@ Status GetBlockBasedTableOptionsFromMap( // @return Status::OK() on success. Otherwise, a non-ok status indicating // error will be returned, and "new_table_options" will be set to // "table_options". +Status GetPlainTableOptionsFromMap( + const ConfigOptions& config_options, const PlainTableOptions& table_options, + const std::unordered_map& opts_map, + PlainTableOptions* new_table_options); Status GetPlainTableOptionsFromMap( const PlainTableOptions& table_options, const std::unordered_map& opts_map, PlainTableOptions* new_table_options, bool input_strings_escaped = false, bool ignore_unknown_options = false); -// Take a string representation of option names and values, apply them into the +// Take a string representation of option names and values, apply them into the // base_options, and return the new options as a result. The string has the // following format: // "write_buffer_size=1024;max_write_buffer_number=2" @@ -277,22 +374,43 @@ Status GetPlainTableOptionsFromMap( // BlockBasedTableOptions as part of the string for block-based table factory: // "write_buffer_size=1024;block_based_table_factory={block_size=4k};" // "max_write_buffer_num=2" +// +// +// The GetColumnFamilyOptionsFromString(ConfigOptions, ...) should be used; the +// alternative signature may be deprecated in a future release. The equivalent +// functionality can be achieved by setting the corresponding options in +// the ConfigOptions parameter. +Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options, + const ColumnFamilyOptions& base_options, + const std::string& opts_str, + ColumnFamilyOptions* new_options); Status GetColumnFamilyOptionsFromString(const ColumnFamilyOptions& base_options, const std::string& opts_str, ColumnFamilyOptions* new_options); +Status GetDBOptionsFromString(const ConfigOptions& config_options, + const DBOptions& base_options, + const std::string& opts_str, + DBOptions* new_options); + Status GetDBOptionsFromString(const DBOptions& base_options, const std::string& opts_str, DBOptions* new_options); +Status GetStringFromDBOptions(const ConfigOptions& config_options, + const DBOptions& db_options, + std::string* opts_str); + Status GetStringFromDBOptions(std::string* opts_str, const DBOptions& db_options, const std::string& delimiter = "; "); +Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options, + const ColumnFamilyOptions& cf_options, + std::string* opts_str); Status GetStringFromColumnFamilyOptions(std::string* opts_str, const ColumnFamilyOptions& cf_options, const std::string& delimiter = "; "); - Status GetStringFromCompressionType(std::string* compression_str, CompressionType compression_type); @@ -301,10 +419,18 @@ std::vector GetSupportedCompressions(); Status GetBlockBasedTableOptionsFromString( const BlockBasedTableOptions& table_options, const std::string& opts_str, BlockBasedTableOptions* new_table_options); +Status GetBlockBasedTableOptionsFromString( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, const std::string& opts_str, + BlockBasedTableOptions* new_table_options); Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options, const std::string& opts_str, PlainTableOptions* new_table_options); +Status GetPlainTableOptionsFromString(const ConfigOptions& config_options, + const PlainTableOptions& table_options, + const std::string& opts_str, + PlainTableOptions* new_table_options); Status GetMemTableRepFactoryFromString( const std::string& opts_str, @@ -312,6 +438,9 @@ Status GetMemTableRepFactoryFromString( Status GetOptionsFromString(const Options& base_options, const std::string& opts_str, Options* new_options); +Status GetOptionsFromString(const ConfigOptions& config_options, + const Options& base_options, + const std::string& opts_str, Options* new_options); Status StringToMap(const std::string& opts_str, std::unordered_map* opts_map); diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 7963513e0..86a6e1d65 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -25,20 +25,24 @@ #include "rocksdb/cache.h" #include "rocksdb/env.h" #include "rocksdb/iterator.h" -#include "rocksdb/options.h" #include "rocksdb/status.h" namespace ROCKSDB_NAMESPACE { // -- Block-based Table +class FilterPolicy; class FlushBlockPolicyFactory; class PersistentCache; class RandomAccessFile; struct TableReaderOptions; struct TableBuilderOptions; class TableBuilder; +class TableFactory; class TableReader; class WritableFileWriter; +struct ColumnFamilyOptions; +struct ConfigOptions; +struct DBOptions; struct EnvOptions; struct Options; @@ -555,8 +559,8 @@ class TableFactory { // RocksDB prints configurations at DB Open(). virtual std::string GetPrintableTableOptions() const = 0; - virtual Status GetOptionString(std::string* /*opt_string*/, - const std::string& /*delimiter*/) const { + virtual Status GetOptionString(const ConfigOptions& /*config_options*/, + std::string* /*opt_string*/) const { return Status::NotSupported( "The table factory doesn't implement GetOptionString()."); } diff --git a/include/rocksdb/utilities/ldb_cmd.h b/include/rocksdb/utilities/ldb_cmd.h index 94548b538..0942db221 100644 --- a/include/rocksdb/utilities/ldb_cmd.h +++ b/include/rocksdb/utilities/ldb_cmd.h @@ -9,6 +9,7 @@ #include #include + #include #include #include @@ -16,6 +17,7 @@ #include #include +#include "rocksdb/convenience.h" #include "rocksdb/env.h" #include "rocksdb/iterator.h" #include "rocksdb/ldb_tool.h" @@ -161,8 +163,6 @@ class LDBCommand { // If true, try to construct options from DB's option files. bool try_load_options_; - bool ignore_unknown_options_; - bool create_if_missing_; /** @@ -237,6 +237,7 @@ class LDBCommand { Options options_; std::vector column_families_; + ConfigOptions config_options_; LDBOptions ldb_options_; private: diff --git a/include/rocksdb/utilities/options_util.h b/include/rocksdb/utilities/options_util.h index 1a29464a6..681b42cfd 100644 --- a/include/rocksdb/utilities/options_util.h +++ b/include/rocksdb/utilities/options_util.h @@ -11,12 +11,14 @@ #include #include +#include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/options.h" #include "rocksdb/status.h" namespace ROCKSDB_NAMESPACE { +struct ConfigOptions; // Constructs the DBOptions and ColumnFamilyDescriptors by loading the // latest RocksDB options file stored in the specified rocksdb database. // @@ -45,13 +47,19 @@ namespace ROCKSDB_NAMESPACE { // pointer options of BlockBasedTableOptions (flush_block_policy_factory, // block_cache, and block_cache_compressed), which will be initialized with // default values. Developers can further specify these three options by -// casting the return value of TableFactoroy::GetOptions() to +// casting the return value of TableFactory::GetOptions() to // BlockBasedTableOptions and making necessary changes. // // ignore_unknown_options can be set to true if you want to ignore options // that are from a newer version of the db, esentially for forward // compatibility. // +// config_options contains a set of options that controls the processing +// of the options. The LoadLatestOptions(ConfigOptions...) should be preferred; +// the alternative signature may be deprecated in a future release. The +// equivalent functionality can be achieved by setting the corresponding options +// in the ConfigOptions parameter. +// // examples/options_file_example.cc demonstrates how to use this function // to open a RocksDB instance. // @@ -67,16 +75,30 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env, std::vector* cf_descs, bool ignore_unknown_options = false, std::shared_ptr* cache = {}); +Status LoadLatestOptions(const ConfigOptions& config_options, + const std::string& dbpath, DBOptions* db_options, + std::vector* cf_descs, + std::shared_ptr* cache = {}); // Similar to LoadLatestOptions, this function constructs the DBOptions // and ColumnFamilyDescriptors based on the specified RocksDB Options file. // +// The LoadOptionsFile(ConfigOptions...) should be preferred; +// the alternative signature may be deprecated in a future release. The +// equivalent functionality can be achieved by setting the corresponding +// options in the ConfigOptions parameter. +// // @see LoadLatestOptions Status LoadOptionsFromFile(const std::string& options_file_name, Env* env, DBOptions* db_options, std::vector* cf_descs, bool ignore_unknown_options = false, std::shared_ptr* cache = {}); +Status LoadOptionsFromFile(const ConfigOptions& config_options, + const std::string& options_file_name, + DBOptions* db_options, + std::vector* cf_descs, + std::shared_ptr* cache = {}); // Returns the latest options file name under the specified db path. Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, @@ -97,6 +119,10 @@ Status CheckOptionsCompatibility( const std::string& dbpath, Env* env, const DBOptions& db_options, const std::vector& cf_descs, bool ignore_unknown_options = false); +Status CheckOptionsCompatibility( + const ConfigOptions& config_options, const std::string& dbpath, + const DBOptions& db_options, + const std::vector& cf_descs); } // namespace ROCKSDB_NAMESPACE #endif // !ROCKSDB_LITE diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 8fc7bea9b..828803bfb 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -26,6 +26,7 @@ set(JNI_NATIVE_SOURCES rocksjni/comparator.cc rocksjni/comparatorjnicallback.cc rocksjni/compression_options.cc + rocksjni/config_options.cc rocksjni/env.cc rocksjni/env_options.cc rocksjni/filter.cc @@ -128,6 +129,7 @@ set(JAVA_MAIN_CLASSES src/main/java/org/rocksdb/ComparatorType.java src/main/java/org/rocksdb/CompressionOptions.java src/main/java/org/rocksdb/CompressionType.java + src/main/java/org/rocksdb/ConfigOptions.java src/main/java/org/rocksdb/DataBlockIndexType.java src/main/java/org/rocksdb/DBOptionsInterface.java src/main/java/org/rocksdb/DBOptions.java @@ -191,6 +193,7 @@ set(JAVA_MAIN_CLASSES src/main/java/org/rocksdb/RocksMemEnv.java src/main/java/org/rocksdb/RocksMutableObject.java src/main/java/org/rocksdb/RocksObject.java + src/main/java/org/rocksdb/SanityLevel.java src/main/java/org/rocksdb/SizeApproximationFlag.java src/main/java/org/rocksdb/SkipListMemTableConfig.java src/main/java/org/rocksdb/Slice.java diff --git a/java/Makefile b/java/Makefile index c2600cf98..12eb95f03 100644 --- a/java/Makefile +++ b/java/Makefile @@ -25,6 +25,7 @@ NATIVE_JAVA_CLASSES = \ org.rocksdb.CompactRangeOptions\ org.rocksdb.ComparatorOptions\ org.rocksdb.CompressionOptions\ + org.rocksdb.ConfigOptions\ org.rocksdb.DBOptions\ org.rocksdb.DirectSlice\ org.rocksdb.Env\ diff --git a/java/rocksjni/config_options.cc b/java/rocksjni/config_options.cc new file mode 100644 index 000000000..299d69815 --- /dev/null +++ b/java/rocksjni/config_options.cc @@ -0,0 +1,88 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::ConfigOptions methods +// from Java side. + +#include + +#include "include/org_rocksdb_ConfigOptions.h" +#include "rocksdb/convenience.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_ConfigOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ConfigOptions_disposeInternal(JNIEnv *, jobject, + jlong jhandle) { + auto *co = reinterpret_cast(jhandle); + assert(co != nullptr); + delete co; +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: newConfigOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_ConfigOptions_newConfigOptions(JNIEnv *, jclass) { + auto *cfg_opt = new ROCKSDB_NAMESPACE::ConfigOptions(); + return reinterpret_cast(cfg_opt); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setDelimiter + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_ConfigOptions_setDelimiter(JNIEnv *env, jclass, + jlong handle, jstring s) { + auto *cfg_opt = reinterpret_cast(handle); + const char *delim = env->GetStringUTFChars(s, nullptr); + if (delim == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + cfg_opt->delimiter = delim; + env->ReleaseStringUTFChars(s, delim); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setIgnoreUnknownOptions + * Signature: (JZ)V + */ +void Java_org_rocksdb_ConfigOptions_setIgnoreUnknownOptions(JNIEnv *, jclass, + jlong handle, + jboolean b) { + auto *cfg_opt = reinterpret_cast(handle); + cfg_opt->ignore_unknown_options = static_cast(b); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setInputStringsEscaped + * Signature: (JZ)V + */ +void Java_org_rocksdb_ConfigOptions_setInputStringsEscaped(JNIEnv *, jclass, + jlong handle, + jboolean b) { + auto *cfg_opt = reinterpret_cast(handle); + cfg_opt->input_strings_escaped = static_cast(b); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setSanityLevel + * Signature: (JI)V + */ +void Java_org_rocksdb_ConfigOptions_setSanityLevel(JNIEnv *, jclass, + jlong handle, jbyte level) { + auto *cfg_opt = reinterpret_cast(handle); + cfg_opt->sanity_level = ROCKSDB_NAMESPACE::SanityLevelJni::toCppSanityLevel(level); +} diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index c13613373..dbc932279 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -3274,12 +3274,46 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptionsFromOptions( return reinterpret_cast(new_opt); } +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: getColumnFamilyOptionsFromProps + * Signature: (JLjava/lang/String;)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__JLjava_lang_String_2( + JNIEnv* env, jclass, jlong cfg_handle, jstring jopt_string) { + const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); + if (opt_string == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + auto* config_options = + reinterpret_cast(cfg_handle); + auto* cf_options = new ROCKSDB_NAMESPACE::ColumnFamilyOptions(); + ROCKSDB_NAMESPACE::Status status = + ROCKSDB_NAMESPACE::GetColumnFamilyOptionsFromString( + *config_options, ROCKSDB_NAMESPACE::ColumnFamilyOptions(), opt_string, + cf_options); + + env->ReleaseStringUTFChars(jopt_string, opt_string); + + // Check if ColumnFamilyOptions creation was possible. + jlong ret_value = 0; + if (status.ok()) { + ret_value = reinterpret_cast(cf_options); + } else { + // if operation failed the ColumnFamilyOptions need to be deleted + // again to prevent a memory leak. + delete cf_options; + } + return ret_value; +} + /* * Class: org_rocksdb_ColumnFamilyOptions * Method: getColumnFamilyOptionsFromProps * Signature: (Ljava/util/String;)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps( +jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__Ljava_lang_String_2( JNIEnv* env, jclass, jstring jopt_string) { const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); if (opt_string == nullptr) { @@ -4667,12 +4701,45 @@ jlong Java_org_rocksdb_DBOptions_newDBOptionsFromOptions( return reinterpret_cast(new_opt); } +/* + * Class: org_rocksdb_DBOptions + * Method: getDBOptionsFromProps + * Signature: (JLjava/lang/String;)J + */ +jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__JLjava_lang_String_2( + JNIEnv* env, jclass, jlong config_handle, jstring jopt_string) { + const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); + if (opt_string == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + auto* config_options = + reinterpret_cast(config_handle); + auto* db_options = new ROCKSDB_NAMESPACE::DBOptions(); + ROCKSDB_NAMESPACE::Status status = ROCKSDB_NAMESPACE::GetDBOptionsFromString( + *config_options, ROCKSDB_NAMESPACE::DBOptions(), opt_string, db_options); + + env->ReleaseStringUTFChars(jopt_string, opt_string); + + // Check if DBOptions creation was possible. + jlong ret_value = 0; + if (status.ok()) { + ret_value = reinterpret_cast(db_options); + } else { + // if operation failed the DBOptions need to be deleted + // again to prevent a memory leak. + delete db_options; + } + return ret_value; +} + /* * Class: org_rocksdb_DBOptions * Method: getDBOptionsFromProps * Signature: (Ljava/util/String;)J */ -jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps( +jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__Ljava_lang_String_2( JNIEnv* env, jclass, jstring jopt_string) { const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); if (opt_string == nullptr) { diff --git a/java/rocksjni/options_util.cc b/java/rocksjni/options_util.cc index e195adafa..f529c548b 100644 --- a/java/rocksjni/options_util.cc +++ b/java/rocksjni/options_util.cc @@ -55,7 +55,7 @@ void build_column_family_descriptor_list( * Method: loadLatestOptions * Signature: (Ljava/lang/String;JLjava/util/List;Z)V */ -void Java_org_rocksdb_OptionsUtil_loadLatestOptions( +void Java_org_rocksdb_OptionsUtil_loadLatestOptions__Ljava_lang_String_2JJLjava_util_List_2Z( JNIEnv* env, jclass /*jcls*/, jstring jdbpath, jlong jenv_handle, jlong jdb_opts_handle, jobject jcfds, jboolean ignore_unknown_options) { jboolean has_exception = JNI_FALSE; @@ -78,12 +78,42 @@ void Java_org_rocksdb_OptionsUtil_loadLatestOptions( } } +/* + * Class: org_rocksdb_OptionsUtil + * Method: loadLatestOptions_1 + * Signature: (JLjava/lang/String;JLjava/util/List;)V + */ +void Java_org_rocksdb_OptionsUtil_loadLatestOptions__JLjava_lang_String_2JLjava_util_List_2( + JNIEnv* env, jclass /*jcls*/, jlong cfg_handle, jstring jdbpath, + jlong jdb_opts_handle, jobject jcfds) { + jboolean has_exception = JNI_FALSE; + auto db_path = + ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jdbpath, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + std::vector cf_descs; + auto* config_options = + reinterpret_cast(cfg_handle); + auto* db_options = + reinterpret_cast(jdb_opts_handle); + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::LoadLatestOptions( + *config_options, db_path, db_options, &cf_descs); + if (!s.ok()) { + // error, raise an exception + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } else { + build_column_family_descriptor_list(env, jcfds, cf_descs); + } +} + /* * Class: org_rocksdb_OptionsUtil * Method: loadOptionsFromFile * Signature: (Ljava/lang/String;JJLjava/util/List;Z)V */ -void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile( +void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile__Ljava_lang_String_2JJLjava_util_List_2Z( JNIEnv* env, jclass /*jcls*/, jstring jopts_file_name, jlong jenv_handle, jlong jdb_opts_handle, jobject jcfds, jboolean ignore_unknown_options) { jboolean has_exception = JNI_FALSE; @@ -106,6 +136,36 @@ void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile( } } +/* + * Class: org_rocksdb_OptionsUtil + * Method: loadOptionsFromFile + * Signature: (JLjava/lang/String;JLjava/util/List;)V + */ +void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile__JLjava_lang_String_2JLjava_util_List_2( + JNIEnv* env, jclass /*jcls*/, jlong cfg_handle, jstring jopts_file_name, + jlong jdb_opts_handle, jobject jcfds) { + jboolean has_exception = JNI_FALSE; + auto opts_file_name = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, jopts_file_name, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + std::vector cf_descs; + auto* config_options = + reinterpret_cast(cfg_handle); + auto* db_options = + reinterpret_cast(jdb_opts_handle); + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::LoadOptionsFromFile( + *config_options, opts_file_name, db_options, &cf_descs); + if (!s.ok()) { + // error, raise an exception + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } else { + build_column_family_descriptor_list(env, jcfds, cf_descs); + } +} + /* * Class: org_rocksdb_OptionsUtil * Method: getLatestOptionsFileName diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index deb88af45..90e1a0977 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -22,6 +22,7 @@ #include #include +#include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/filter_policy.h" #include "rocksdb/rate_limiter.h" @@ -7530,5 +7531,41 @@ class ReusedSynchronisationTypeJni { } } }; +// The portal class for org.rocksdb.SanityLevel +class SanityLevelJni { + public: + // Returns the equivalent org.rocksdb.SanityLevel for the provided + // C++ ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel enum + static jbyte toJavaSanityLevel( + const ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel &sanity_level) { + switch (sanity_level) { + case ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel::kSanityLevelNone: + return 0x0; + case ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel:: + kSanityLevelLooselyCompatible: + return 0x1; + case ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel:: + kSanityLevelExactMatch: + return -0x01; + default: + return -0x01; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel enum for + // the provided Java org.rocksdb.SanityLevel + static ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel toCppSanityLevel( + jbyte sanity_level) { + switch (sanity_level) { + case 0x0: + return ROCKSDB_NAMESPACE::ConfigOptions::kSanityLevelNone; + case 0x1: + return ROCKSDB_NAMESPACE::ConfigOptions::kSanityLevelLooselyCompatible; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ConfigOptions::kSanityLevelExactMatch; + } + } +}; } // namespace ROCKSDB_NAMESPACE #endif // JAVA_ROCKSJNI_PORTAL_H_ diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java index 07f07b3ec..f7852d37d 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -96,20 +96,40 @@ public class ColumnFamilyOptions extends RocksObject */ public static ColumnFamilyOptions getColumnFamilyOptionsFromProps( final Properties properties) { - if (properties == null || properties.size() == 0) { - throw new IllegalArgumentException( - "Properties value must contain at least one value."); - } ColumnFamilyOptions columnFamilyOptions = null; - StringBuilder stringBuilder = new StringBuilder(); - for (final String name : properties.stringPropertyNames()){ - stringBuilder.append(name); - stringBuilder.append("="); - stringBuilder.append(properties.getProperty(name)); - stringBuilder.append(";"); + final long handle = + getColumnFamilyOptionsFromProps(Options.getOptionStringFromProps(properties)); + if (handle != 0) { + columnFamilyOptions = new ColumnFamilyOptions(handle); } - long handle = getColumnFamilyOptionsFromProps( - stringBuilder.toString()); + return columnFamilyOptions; + } + + /** + *

Method to get a options instance by using pre-configured + * property values. If one or many values are undefined in + * the context of RocksDB the method will return a null + * value.

+ * + *

Note: Property keys can be derived from + * getter methods within the options class. Example: the method + * {@code writeBufferSize()} has a property key: + * {@code write_buffer_size}.

+ * + * @param cfgOpts ConfigOptions controlling how the properties are parsed. + * @param properties {@link java.util.Properties} instance. + * + * @return {@link org.rocksdb.ColumnFamilyOptions instance} + * or null. + * + * @throws java.lang.IllegalArgumentException if null or empty + * {@link Properties} instance is passed to the method call. + */ + public static ColumnFamilyOptions getColumnFamilyOptionsFromProps( + final ConfigOptions cfgOpts, final Properties properties) { + ColumnFamilyOptions columnFamilyOptions = null; + final long handle = getColumnFamilyOptionsFromProps( + cfgOpts.nativeHandle_, Options.getOptionStringFromProps(properties)); if (handle != 0){ columnFamilyOptions = new ColumnFamilyOptions(handle); } @@ -825,7 +845,8 @@ public class ColumnFamilyOptions extends RocksObject } private static native long getColumnFamilyOptionsFromProps( - String optString); + final long cfgHandle, String optString); + private static native long getColumnFamilyOptionsFromProps(final String optString); private static native long newColumnFamilyOptions(); private static native long copyColumnFamilyOptions(final long handle); diff --git a/java/src/main/java/org/rocksdb/ConfigOptions.java b/java/src/main/java/org/rocksdb/ConfigOptions.java new file mode 100644 index 000000000..83f65db5c --- /dev/null +++ b/java/src/main/java/org/rocksdb/ConfigOptions.java @@ -0,0 +1,47 @@ +package org.rocksdb; + +public class ConfigOptions extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + /** + * Construct with default Options + */ + public ConfigOptions() { + super(newConfigOptions()); + } + + public ConfigOptions setDelimiter(final String delimiter) { + setDelimiter(nativeHandle_, delimiter); + return this; + } + public ConfigOptions setIgnoreUnknownOptions(final boolean ignore) { + setIgnoreUnknownOptions(nativeHandle_, ignore); + return this; + } + + public ConfigOptions setEnv(final Env env) { + setEnv(nativeHandle_, env.nativeHandle_); + return this; + } + + public ConfigOptions setInputStringsEscaped(final boolean escaped) { + setInputStringsEscaped(nativeHandle_, escaped); + return this; + } + + public ConfigOptions setSanityLevel(final SanityLevel level) { + setSanityLevel(nativeHandle_, level.getValue()); + return this; + } + + @Override protected final native void disposeInternal(final long handle); + + private native static long newConfigOptions(); + private native static void setEnv(final long handle, final long envHandle); + private native static void setDelimiter(final long handle, final String delimiter); + private native static void setIgnoreUnknownOptions(final long handle, final boolean ignore); + private native static void setInputStringsEscaped(final long handle, final boolean escaped); + private native static void setSanityLevel(final long handle, final byte level); +} diff --git a/java/src/main/java/org/rocksdb/DBOptions.java b/java/src/main/java/org/rocksdb/DBOptions.java index 36cc4abd9..eab4ab2a2 100644 --- a/java/src/main/java/org/rocksdb/DBOptions.java +++ b/java/src/main/java/org/rocksdb/DBOptions.java @@ -71,6 +71,7 @@ public class DBOptions extends RocksObject * {@code allowMmapReads()} has a property key: * {@code allow_mmap_reads}.

* + * @param cfgOpts The ConfigOptions to control how the string is processed. * @param properties {@link java.util.Properties} instance. * * @return {@link org.rocksdb.DBOptions instance} @@ -80,22 +81,40 @@ public class DBOptions extends RocksObject * {@link java.util.Properties} instance is passed to the method call. */ public static DBOptions getDBOptionsFromProps( - final Properties properties) { - if (properties == null || properties.size() == 0) { - throw new IllegalArgumentException( - "Properties value must contain at least one value."); - } + final ConfigOptions cfgOpts, final Properties properties) { DBOptions dbOptions = null; - StringBuilder stringBuilder = new StringBuilder(); - for (final String name : properties.stringPropertyNames()){ - stringBuilder.append(name); - stringBuilder.append("="); - stringBuilder.append(properties.getProperty(name)); - stringBuilder.append(";"); + final String optionsString = Options.getOptionStringFromProps(properties); + final long handle = getDBOptionsFromProps(cfgOpts.nativeHandle_, optionsString); + if (handle != 0) { + dbOptions = new DBOptions(handle); } - long handle = getDBOptionsFromProps( - stringBuilder.toString()); - if (handle != 0){ + return dbOptions; + } + + /** + *

Method to get a options instance by using pre-configured + * property values. If one or many values are undefined in + * the context of RocksDB the method will return a null + * value.

+ * + *

Note: Property keys can be derived from + * getter methods within the options class. Example: the method + * {@code allowMmapReads()} has a property key: + * {@code allow_mmap_reads}.

+ * + * @param properties {@link java.util.Properties} instance. + * + * @return {@link org.rocksdb.DBOptions instance} + * or null. + * + * @throws java.lang.IllegalArgumentException if null or empty + * {@link java.util.Properties} instance is passed to the method call. + */ + public static DBOptions getDBOptionsFromProps(final Properties properties) { + DBOptions dbOptions = null; + final String optionsString = Options.getOptionStringFromProps(properties); + final long handle = getDBOptionsFromProps(optionsString); + if (handle != 0) { dbOptions = new DBOptions(handle); } return dbOptions; @@ -1175,8 +1194,8 @@ public class DBOptions extends RocksObject super(nativeHandle); } - private static native long getDBOptionsFromProps( - String optString); + private static native long getDBOptionsFromProps(long cfgHandle, String optString); + private static native long getDBOptionsFromProps(String optString); private static native long newDBOptions(); private static native long copyDBOptions(final long handle); diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index 8a534f503..349e191ee 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -6,10 +6,7 @@ package org.rocksdb; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; +import java.util.*; /** * Options to control the behavior of a database. It will be used @@ -27,6 +24,25 @@ public class Options extends RocksObject RocksDB.loadLibrary(); } + /** + * Converts the input properties into a Options-style formatted string + * @param properties The set of properties to convert + * @return The Options-style representation of those properties. + */ + public static String getOptionStringFromProps(final Properties properties) { + if (properties == null || properties.size() == 0) { + throw new IllegalArgumentException("Properties value must contain at least one value."); + } + StringBuilder stringBuilder = new StringBuilder(); + for (final String name : properties.stringPropertyNames()) { + stringBuilder.append(name); + stringBuilder.append("="); + stringBuilder.append(properties.getProperty(name)); + stringBuilder.append(";"); + } + return stringBuilder.toString(); + } + /** * Construct options for opening a RocksDB. * diff --git a/java/src/main/java/org/rocksdb/OptionsUtil.java b/java/src/main/java/org/rocksdb/OptionsUtil.java index f153556ba..4f575f216 100644 --- a/java/src/main/java/org/rocksdb/OptionsUtil.java +++ b/java/src/main/java/org/rocksdb/OptionsUtil.java @@ -59,7 +59,7 @@ public class OptionsUtil { * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be * returned. * @param ignoreUnknownOptions this flag can be set to true if you want to - * ignore options that are from a newer version of the db, esentially for + * ignore options that are from a newer version of the db, essentially for * forward compatibility. * * @throws RocksDBException thrown if error happens in underlying @@ -71,6 +71,25 @@ public class OptionsUtil { dbPath, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions); } + /** + * Similar to LoadLatestOptions, this function constructs the DBOptions + * and ColumnFamilyDescriptors based on the specified RocksDB Options file. + * See LoadLatestOptions above. + * + * @param dbPath the path to the RocksDB. + * @param configOptions {@link org.rocksdb.ConfigOptions} instance. + * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be + * filled and returned. + * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be + * returned. + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static void loadLatestOptions(ConfigOptions configOptions, String dbPath, + DBOptions dbOptions, List cfDescs) throws RocksDBException { + loadLatestOptions(configOptions.nativeHandle_, dbPath, dbOptions.nativeHandle_, cfDescs); + } + /** * Similar to LoadLatestOptions, this function constructs the DBOptions * and ColumnFamilyDescriptors based on the specified RocksDB Options file. @@ -111,6 +130,26 @@ public class OptionsUtil { optionsFileName, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions); } + /** + * Similar to LoadLatestOptions, this function constructs the DBOptions + * and ColumnFamilyDescriptors based on the specified RocksDB Options file. + * See LoadLatestOptions above. + * + * @param optionsFileName the RocksDB options file path. + * @param configOptions {@link org.rocksdb.ConfigOptions} instance. + * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be + * filled and returned. + * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be + * returned. + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static void loadOptionsFromFile(ConfigOptions configOptions, String optionsFileName, + DBOptions dbOptions, List cfDescs) throws RocksDBException { + loadOptionsFromFile( + configOptions.nativeHandle_, optionsFileName, dbOptions.nativeHandle_, cfDescs); + } + /** * Returns the latest options file name under the specified RocksDB path. * @@ -134,9 +173,13 @@ public class OptionsUtil { // native methods private native static void loadLatestOptions(String dbPath, long envHandle, long dbOptionsHandle, List cfDescs, boolean ignoreUnknownOptions) throws RocksDBException; + private native static void loadLatestOptions(long cfgHandle, String dbPath, long dbOptionsHandle, + List cfDescs) throws RocksDBException; private native static void loadOptionsFromFile(String optionsFileName, long envHandle, long dbOptionsHandle, List cfDescs, boolean ignoreUnknownOptions) throws RocksDBException; + private native static void loadOptionsFromFile(long cfgHandle, String optionsFileName, + long dbOptionsHandle, List cfDescs) throws RocksDBException; private native static String getLatestOptionsFileName(String dbPath, long envHandle) throws RocksDBException; } diff --git a/java/src/main/java/org/rocksdb/SanityLevel.java b/java/src/main/java/org/rocksdb/SanityLevel.java new file mode 100644 index 000000000..775611c07 --- /dev/null +++ b/java/src/main/java/org/rocksdb/SanityLevel.java @@ -0,0 +1,42 @@ +package org.rocksdb; + +public enum SanityLevel { + NONE((byte) 0x0), + LOOSELY_COMPATIBLE((byte) 0x1), + EXACT_MATCH((byte) 0xFF); + + private final byte value; + + SanityLevel(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value. + */ + // TODO(AR) should be made package-private + public byte getValue() { + return value; + } + + /** + * Get the SanityLevel from the internal representation value. + * + * @param value the internal representation value. + * + * @return the SanityLevel + * + * @throws IllegalArgumentException if the value does not match a + * SanityLevel + */ + static SanityLevel fromValue(final byte value) throws IllegalArgumentException { + for (final SanityLevel level : SanityLevel.values()) { + if (level.value == value) { + return level; + } + } + throw new IllegalArgumentException("Unknown value for SanityLevel: " + value); + } +} diff --git a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java index af67f4663..df01ee60b 100644 --- a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java @@ -54,6 +54,27 @@ public class ColumnFamilyOptionsTest { } } + @Test + public void getColumnFamilyOptionsFromPropsWithIgnoreIllegalValue() { + // setup sample properties + final Properties properties = new Properties(); + properties.put("tomato", "1024"); + properties.put("burger", "2"); + properties.put("write_buffer_size", "112"); + properties.put("max_write_buffer_number", "13"); + + try (final ConfigOptions cfgOpts = new ConfigOptions().setIgnoreUnknownOptions(true); + final ColumnFamilyOptions opt = + ColumnFamilyOptions.getColumnFamilyOptionsFromProps(cfgOpts, properties)) { + // setup sample properties + assertThat(opt).isNotNull(); + assertThat(String.valueOf(opt.writeBufferSize())) + .isEqualTo(properties.get("write_buffer_size")); + assertThat(String.valueOf(opt.maxWriteBufferNumber())) + .isEqualTo(properties.get("max_write_buffer_number")); + } + } + @Test public void failColumnFamilyOptionsFromPropsWithIllegalValue() { // setup sample properties diff --git a/options/cf_options.cc b/options/cf_options.cc index 65b198510..ec1267030 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -347,12 +347,13 @@ std::unordered_map OptionTypeFlags::kMutable, offsetof(struct MutableCFOptions, sample_for_compression)}}}; -Status ParseColumnFamilyOption(const std::string& name, +Status ParseColumnFamilyOption(const ConfigOptions& config_options, + const std::string& name, const std::string& org_value, - ColumnFamilyOptions* new_options, - bool input_strings_escaped) { - const std::string& value = - input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + ColumnFamilyOptions* new_options) { + const std::string& value = config_options.input_strings_escaped + ? UnescapeOptionString(org_value) + : org_value; try { if (name == "block_based_table_factory") { // Nested options diff --git a/options/options_helper.cc b/options/options_helper.cc index e86925d5a..879697eb9 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -10,6 +10,7 @@ #include #include +#include "options/options_type.h" #include "rocksdb/cache.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/convenience.h" @@ -353,10 +354,14 @@ bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str, } static bool SerializeStruct( - const void* const options, std::string* value, + const void* const opt_ptr, std::string* value, const std::unordered_map& type_info_map) { + ConfigOptions config_options; + config_options.delimiter = ";"; + std::string opt_str; - Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";"); + Status s = + GetStringFromStruct(config_options, opt_ptr, type_info_map, &opt_str); if (!s.ok()) { return false; } @@ -1010,9 +1015,9 @@ Status StringToMap(const std::string& opts_str, } Status GetStringFromStruct( - std::string* opt_string, const void* const options, + const ConfigOptions& cfg_options, const void* const opt_ptr, const std::unordered_map& type_info, - const std::string& delimiter) { + std::string* opt_string) { assert(opt_string); opt_string->clear(); for (const auto iter : type_info) { @@ -1023,12 +1028,12 @@ Status GetStringFromStruct( continue; } const char* opt_address = - reinterpret_cast(options) + opt_info.offset; + reinterpret_cast(opt_ptr) + opt_info.offset; std::string value; bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); if (result) { - opt_string->append(iter.first + "=" + value + delimiter); + opt_string->append(iter.first + "=" + value + cfg_options.delimiter); } else { return Status::InvalidArgument("failed to serialize %s\n", iter.first.c_str()); @@ -1040,15 +1045,32 @@ Status GetStringFromStruct( Status GetStringFromDBOptions(std::string* opt_string, const DBOptions& db_options, const std::string& delimiter) { - return GetStringFromStruct(opt_string, &db_options, db_options_type_info, - delimiter); + ConfigOptions config_options; + config_options.delimiter = delimiter; + return GetStringFromDBOptions(config_options, db_options, opt_string); +} + +Status GetStringFromDBOptions(const ConfigOptions& cfg_options, + const DBOptions& db_options, + std::string* opt_string) { + return GetStringFromStruct(cfg_options, &db_options, db_options_type_info, + opt_string); } Status GetStringFromColumnFamilyOptions(std::string* opt_string, const ColumnFamilyOptions& cf_options, const std::string& delimiter) { - return GetStringFromStruct(opt_string, &cf_options, cf_options_type_info, - delimiter); + ConfigOptions config_options; + config_options.delimiter = delimiter; + return GetStringFromColumnFamilyOptions(config_options, cf_options, + opt_string); +} + +Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options, + const ColumnFamilyOptions& cf_options, + std::string* opt_string) { + return GetStringFromStruct(config_options, &cf_options, cf_options_type_info, + opt_string); } Status GetStringFromCompressionType(std::string* compression_str, @@ -1073,12 +1095,13 @@ std::vector GetSupportedCompressions() { return supported_compressions; } -Status ParseDBOption(const std::string& name, - const std::string& org_value, - DBOptions* new_options, - bool input_strings_escaped = false) { - const std::string& value = - input_strings_escaped ? UnescapeOptionString(org_value) : org_value; +static Status ParseDBOption(const ConfigOptions& config_options, + const std::string& name, + const std::string& org_value, + DBOptions* new_options) { + const std::string& value = config_options.input_strings_escaped + ? UnescapeOptionString(org_value) + : org_value; try { if (name == "rate_limiter_bytes_per_sec") { new_options->rate_limiter.reset( @@ -1121,37 +1144,28 @@ Status GetColumnFamilyOptionsFromMap( const std::unordered_map& opts_map, ColumnFamilyOptions* new_options, bool input_strings_escaped, bool ignore_unknown_options) { - return GetColumnFamilyOptionsFromMapInternal( - base_options, opts_map, new_options, input_strings_escaped, nullptr, - ignore_unknown_options); + ConfigOptions config_options; + config_options.ignore_unknown_options = ignore_unknown_options; + config_options.input_strings_escaped = input_strings_escaped; + return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map, + new_options); } -Status GetColumnFamilyOptionsFromMapInternal( +Status GetColumnFamilyOptionsFromMap( + const ConfigOptions& config_options, const ColumnFamilyOptions& base_options, const std::unordered_map& opts_map, - ColumnFamilyOptions* new_options, bool input_strings_escaped, - std::vector* unsupported_options_names, - bool ignore_unknown_options) { + ColumnFamilyOptions* new_options) { assert(new_options); *new_options = base_options; - if (unsupported_options_names) { - unsupported_options_names->clear(); - } for (const auto& o : opts_map) { - auto s = ParseColumnFamilyOption(o.first, o.second, new_options, - input_strings_escaped); + auto s = + ParseColumnFamilyOption(config_options, o.first, o.second, new_options); if (!s.ok()) { if (s.IsNotSupported()) { - // If the deserialization of the specified option is not supported - // and an output vector of unsupported_options is provided, then - // we log the name of the unsupported option and proceed. - if (unsupported_options_names != nullptr) { - unsupported_options_names->push_back(o.first); - } - // Note that we still return Status::OK in such case to maintain - // the backward compatibility in the old public API defined in - // rocksdb/convenience.h - } else if (s.IsInvalidArgument() && ignore_unknown_options) { + continue; + } else if (s.IsInvalidArgument() && + config_options.ignore_unknown_options) { continue; } else { // Restore "new_options" to the default "base_options". @@ -1167,13 +1181,25 @@ Status GetColumnFamilyOptionsFromString( const ColumnFamilyOptions& base_options, const std::string& opts_str, ColumnFamilyOptions* new_options) { + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + return GetColumnFamilyOptionsFromString(config_options, base_options, + opts_str, new_options); +} + +Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options, + const ColumnFamilyOptions& base_options, + const std::string& opts_str, + ColumnFamilyOptions* new_options) { std::unordered_map opts_map; Status s = StringToMap(opts_str, &opts_map); if (!s.ok()) { *new_options = base_options; return s; } - return GetColumnFamilyOptionsFromMap(base_options, opts_map, new_options); + return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map, + new_options); } Status GetDBOptionsFromMap( @@ -1181,25 +1207,33 @@ Status GetDBOptionsFromMap( const std::unordered_map& opts_map, DBOptions* new_options, bool input_strings_escaped, bool ignore_unknown_options) { - return GetDBOptionsFromMapInternal(base_options, opts_map, new_options, - input_strings_escaped, nullptr, - ignore_unknown_options); + ConfigOptions config_options; + config_options.input_strings_escaped = input_strings_escaped; + config_options.ignore_unknown_options = ignore_unknown_options; + return GetDBOptionsFromMap(config_options, base_options, opts_map, + new_options); +} + +Status GetDBOptionsFromMap( + const ConfigOptions& config_options, const DBOptions& base_options, + const std::unordered_map& opts_map, + DBOptions* new_options) { + return GetDBOptionsFromMapInternal(config_options, base_options, opts_map, + new_options, nullptr); } Status GetDBOptionsFromMapInternal( - const DBOptions& base_options, + const ConfigOptions& config_options, const DBOptions& base_options, const std::unordered_map& opts_map, - DBOptions* new_options, bool input_strings_escaped, - std::vector* unsupported_options_names, - bool ignore_unknown_options) { + DBOptions* new_options, + std::vector* unsupported_options_names) { assert(new_options); *new_options = base_options; if (unsupported_options_names) { unsupported_options_names->clear(); } for (const auto& o : opts_map) { - auto s = ParseDBOption(o.first, o.second, - new_options, input_strings_escaped); + auto s = ParseDBOption(config_options, o.first, o.second, new_options); if (!s.ok()) { if (s.IsNotSupported()) { // If the deserialization of the specified option is not supported @@ -1211,7 +1245,8 @@ Status GetDBOptionsFromMapInternal( // Note that we still return Status::OK in such case to maintain // the backward compatibility in the old public API defined in // rocksdb/convenience.h - } else if (s.IsInvalidArgument() && ignore_unknown_options) { + } else if (s.IsInvalidArgument() && + config_options.ignore_unknown_options) { continue; } else { // Restore "new_options" to the default "base_options". @@ -1223,21 +1258,44 @@ Status GetDBOptionsFromMapInternal( return Status::OK(); } -Status GetDBOptionsFromString( - const DBOptions& base_options, - const std::string& opts_str, - DBOptions* new_options) { +Status GetDBOptionsFromString(const DBOptions& base_options, + const std::string& opts_str, + DBOptions* new_options) { + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + return GetDBOptionsFromString(config_options, base_options, opts_str, + new_options); +} + +Status GetDBOptionsFromString(const ConfigOptions& config_options, + const DBOptions& base_options, + const std::string& opts_str, + DBOptions* new_options) { std::unordered_map opts_map; Status s = StringToMap(opts_str, &opts_map); if (!s.ok()) { *new_options = base_options; return s; } - return GetDBOptionsFromMap(base_options, opts_map, new_options); + return GetDBOptionsFromMap(config_options, base_options, opts_map, + new_options); } Status GetOptionsFromString(const Options& base_options, const std::string& opts_str, Options* new_options) { + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + return GetOptionsFromString(config_options, base_options, opts_str, + new_options); +} + +Status GetOptionsFromString(const ConfigOptions& config_options, + const Options& base_options, + const std::string& opts_str, Options* new_options) { std::unordered_map opts_map; Status s = StringToMap(opts_str, &opts_map); if (!s.ok()) { @@ -1246,9 +1304,11 @@ Status GetOptionsFromString(const Options& base_options, DBOptions new_db_options(base_options); ColumnFamilyOptions new_cf_options(base_options); for (const auto& o : opts_map) { - if (ParseDBOption(o.first, o.second, &new_db_options).ok()) { - } else if (ParseColumnFamilyOption( - o.first, o.second, &new_cf_options).ok()) { + if (ParseDBOption(config_options, o.first, o.second, &new_db_options) + .ok()) { + } else if (ParseColumnFamilyOption(config_options, o.first, o.second, + &new_cf_options) + .ok()) { } else { return Status::InvalidArgument("Can't parse option " + o.first); } @@ -1261,13 +1321,22 @@ Status GetTableFactoryFromMap( const std::string& factory_name, const std::unordered_map& opt_map, std::shared_ptr* table_factory, bool ignore_unknown_options) { + ConfigOptions + config_options; // Use default for escaped(true) and check (exact) + config_options.ignore_unknown_options = ignore_unknown_options; + return GetTableFactoryFromMap(config_options, factory_name, opt_map, + table_factory); +} + +Status GetTableFactoryFromMap( + const ConfigOptions& config_options, const std::string& factory_name, + const std::unordered_map& opt_map, + std::shared_ptr* table_factory) { Status s; if (factory_name == BlockBasedTableFactory().Name()) { BlockBasedTableOptions bbt_opt; - s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map, - &bbt_opt, - true, /* input_strings_escaped */ - ignore_unknown_options); + s = GetBlockBasedTableOptionsFromMap( + config_options, BlockBasedTableOptions(), opt_map, &bbt_opt); if (!s.ok()) { return s; } @@ -1275,9 +1344,8 @@ Status GetTableFactoryFromMap( return Status::OK(); } else if (factory_name == PlainTableFactory().Name()) { PlainTableOptions pt_opt; - s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt, - true, /* input_strings_escaped */ - ignore_unknown_options); + s = GetPlainTableOptionsFromMap(config_options, PlainTableOptions(), + opt_map, &pt_opt); if (!s.ok()) { return s; } diff --git a/options/options_helper.h b/options/options_helper.h index 6feca8f3d..399ba1034 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -19,6 +19,7 @@ #include "rocksdb/universal_compaction.h" namespace ROCKSDB_NAMESPACE { +struct ConfigOptions; DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, const MutableDBOptions& mutable_db_options); @@ -28,11 +29,15 @@ ColumnFamilyOptions BuildColumnFamilyOptions( const MutableCFOptions& mutable_cf_options); #ifndef ROCKSDB_LITE +Status GetStringFromStruct( + const ConfigOptions& config_options, const void* const opt_ptr, + const std::unordered_map& type_info, + std::string* opt_string); -Status ParseColumnFamilyOption(const std::string& name, +Status ParseColumnFamilyOption(const ConfigOptions& config_options, + const std::string& name, const std::string& org_value, - ColumnFamilyOptions* new_options, - bool input_strings_escaped = false); + ColumnFamilyOptions* new_options); Status GetMutableOptionsFromStrings( const MutableCFOptions& base_options, @@ -54,6 +59,11 @@ Status ParseCompressionOptions(const std::string& value, const std::string& name, CompressionOptions& compression_opts); +Status GetTableFactoryFromMap( + const ConfigOptions& config_options, const std::string& factory_name, + const std::unordered_map& opt_map, + std::shared_ptr* table_factory); + // A helper function that converts "opt_address" to a std::string // based on the specified OptionType. bool SerializeSingleOptionHelper(const char* opt_address, @@ -63,21 +73,10 @@ bool SerializeSingleOptionHelper(const char* opt_address, // this further takes an optional output vector "unsupported_options_names", // which stores the name of all the unsupported options specified in "opts_map". Status GetDBOptionsFromMapInternal( - const DBOptions& base_options, + const ConfigOptions& config_options, const DBOptions& base_options, const std::unordered_map& opts_map, - DBOptions* new_options, bool input_strings_escaped, - std::vector* unsupported_options_names = nullptr, - bool ignore_unknown_options = false); - -// In addition to its public version defined in rocksdb/convenience.h, -// this further takes an optional output vector "unsupported_options_names", -// which stores the name of all the unsupported options specified in "opts_map". -Status GetColumnFamilyOptionsFromMapInternal( - const ColumnFamilyOptions& base_options, - const std::unordered_map& opts_map, - ColumnFamilyOptions* new_options, bool input_strings_escaped, - std::vector* unsupported_options_names = nullptr, - bool ignore_unknown_options = false); + DBOptions* new_options, + std::vector* unsupported_options_names = nullptr); bool ParseSliceTransform( const std::string& value, @@ -89,10 +88,6 @@ extern Status StringToMap( extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, const std::string& value); -Status GetStringFromStruct( - std::string* opt_string, const void* const options, - const std::unordered_map& type_info, - const std::string& delimiter); #endif // !ROCKSDB_LITE struct OptionsHelper { diff --git a/options/options_parser.cc b/options/options_parser.cc index 0f8eecc9a..50f88db1b 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -16,14 +16,15 @@ #include "file/read_write_util.h" #include "file/writable_file_writer.h" #include "options/options_helper.h" +#include "options/options_sanity_check.h" +#include "port/port.h" #include "rocksdb/convenience.h" #include "rocksdb/db.h" +#include "table/block_based/block_based_table_factory.h" #include "test_util/sync_point.h" #include "util/cast_util.h" #include "util/string_util.h" -#include "port/port.h" - namespace ROCKSDB_NAMESPACE { static const std::string option_file_header = @@ -38,6 +39,25 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, const std::vector& cf_names, const std::vector& cf_opts, const std::string& file_name, FileSystem* fs) { + ConfigOptions + config_options; // Use default for escaped(true) and check (exact) + config_options.delimiter = "\n "; + // If a readahead size was set in the input options, use it + if (db_opt.log_readahead_size > 0) { + config_options.file_readahead_size = db_opt.log_readahead_size; + } + return PersistRocksDBOptions(config_options, db_opt, cf_names, cf_opts, + file_name, fs); +} + +Status PersistRocksDBOptions(const ConfigOptions& config_options_in, + const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs) { + ConfigOptions config_options = config_options_in; + config_options.delimiter = "\n "; // Override the default to nl + TEST_SYNC_POINT("PersistRocksDBOptions:start"); if (cf_names.size() != cf_opts.size()) { return Status::InvalidArgument( @@ -68,7 +88,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] + "]\n "); - s = GetStringFromDBOptions(&options_file_content, db_opt, "\n "); + s = GetStringFromDBOptions(config_options, db_opt, &options_file_content); if (!s.ok()) { writable->Close(); return s; @@ -79,8 +99,8 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, // CFOptions section writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); - s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i], - "\n "); + s = GetStringFromColumnFamilyOptions(config_options, cf_opts[i], + &options_file_content); if (!s.ok()) { writable->Close(); return s; @@ -93,7 +113,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, tf->Name() + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); options_file_content.clear(); - s = tf->GetOptionString(&options_file_content, "\n "); + s = tf->GetOptionString(config_options, &options_file_content); if (!s.ok()) { return s; } @@ -104,7 +124,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, writable->Close(); return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( - db_opt, cf_names, cf_opts, file_name, fs); + config_options, db_opt, cf_names, cf_opts, file_name, fs); } RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); } @@ -205,7 +225,20 @@ Status RocksDBOptionsParser::ParseStatement(std::string* name, Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs, bool ignore_unknown_options, size_t file_readahead_size) { + ConfigOptions + config_options; // Use default for escaped(true) and check (exact) + config_options.ignore_unknown_options = ignore_unknown_options; + if (file_readahead_size > 0) { + config_options.file_readahead_size = file_readahead_size; + } + return Parse(config_options, file_name, fs); +} + +Status RocksDBOptionsParser::Parse(const ConfigOptions& config_options_in, + const std::string& file_name, + FileSystem* fs) { Reset(); + ConfigOptions config_options = config_options_in; std::unique_ptr seq_file; Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file, @@ -213,9 +246,8 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs, if (!s.ok()) { return s; } - SequentialFileReader sf_reader(std::move(seq_file), file_name, - file_readahead_size); + config_options.file_readahead_size); OptionSection section = kOptionSectionUnknown; std::string title; @@ -235,7 +267,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs, continue; } if (IsSection(line)) { - s = EndSection(section, title, argument, opt_map, ignore_unknown_options); + s = EndSection(config_options, section, title, argument, opt_map); opt_map.clear(); if (!s.ok()) { return s; @@ -243,10 +275,11 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs, // If the option file is not generated by a higher minor version, // there shouldn't be any unknown option. - if (ignore_unknown_options && section == kOptionSectionVersion) { + if (config_options.ignore_unknown_options && + section == kOptionSectionVersion) { if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR && db_version[1] <= ROCKSDB_MINOR)) { - ignore_unknown_options = false; + config_options.ignore_unknown_options = false; } } @@ -265,7 +298,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs, } } - s = EndSection(section, title, argument, opt_map, ignore_unknown_options); + s = EndSection(config_options, section, title, argument, opt_map); opt_map.clear(); if (!s.ok()) { return s; @@ -372,14 +405,12 @@ Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name, } Status RocksDBOptionsParser::EndSection( - const OptionSection section, const std::string& section_title, - const std::string& section_arg, - const std::unordered_map& opt_map, - bool ignore_unknown_options) { + const ConfigOptions& config_options, const OptionSection section, + const std::string& section_title, const std::string& section_arg, + const std::unordered_map& opt_map) { Status s; if (section == kOptionSectionDBOptions) { - s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true, - ignore_unknown_options); + s = GetDBOptionsFromMap(config_options, DBOptions(), opt_map, &db_opt_); if (!s.ok()) { return s; } @@ -390,9 +421,8 @@ Status RocksDBOptionsParser::EndSection( assert(GetCFOptions(section_arg) == nullptr); cf_names_.emplace_back(section_arg); cf_opts_.emplace_back(); - s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map, - &cf_opts_.back(), true, - ignore_unknown_options); + s = GetColumnFamilyOptionsFromMap(config_options, ColumnFamilyOptions(), + opt_map, &cf_opts_.back()); if (!s.ok()) { return s; } @@ -409,9 +439,10 @@ Status RocksDBOptionsParser::EndSection( } // Ignore error as table factory deserialization is optional s = GetTableFactoryFromMap( + config_options, section_title.substr( opt_section_titles[kOptionSectionTableOptions].size()), - opt_map, &(cf_opt->table_factory), ignore_unknown_options); + opt_map, &(cf_opt->table_factory)); if (!s.ok()) { return s; } @@ -652,36 +683,27 @@ bool AreEqualOptions( } Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( - const DBOptions& db_opt, const std::vector& cf_names, + const ConfigOptions& config_options, const DBOptions& db_opt, + const std::vector& cf_names, const std::vector& cf_opts, - const std::string& file_name, FileSystem* fs, - OptionsSanityCheckLevel sanity_check_level, bool ignore_unknown_options) { - // We infer option file readhead size from log readahead size. - // If it is not given, use 512KB. - size_t file_readahead_size = db_opt.log_readahead_size; - if (file_readahead_size == 0) { - const size_t kDefaultOptionFileReadAheadSize = 512 * 1024; - file_readahead_size = kDefaultOptionFileReadAheadSize; - } - + const std::string& file_name, FileSystem* fs) { RocksDBOptionsParser parser; - Status s = - parser.Parse(file_name, fs, ignore_unknown_options, file_readahead_size); + Status s = parser.Parse(config_options, file_name, fs); if (!s.ok()) { return s; } // Verify DBOptions - s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(), - sanity_check_level); + s = VerifyDBOptions(config_options, db_opt, *parser.db_opt(), + parser.db_opt_map()); if (!s.ok()) { return s; } // Verify ColumnFamily Name if (cf_names.size() != parser.cf_names()->size()) { - if (sanity_check_level >= - OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) { + if (config_options.sanity_level >= + ConfigOptions::kSanityLevelLooselyCompatible) { return Status::InvalidArgument( "[RocksDBOptionParser Error] The persisted options does not have " "the same number of column family names as the db instance."); @@ -703,8 +725,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( // Verify Column Family Options if (cf_opts.size() != parser.cf_opts()->size()) { - if (sanity_check_level >= - OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) { + if (config_options.sanity_level >= + ConfigOptions::kSanityLevelLooselyCompatible) { return Status::InvalidArgument( "[RocksDBOptionsParser Error]", "The persisted options does not have the same number of " @@ -717,14 +739,13 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( } } for (size_t i = 0; i < cf_opts.size(); ++i) { - s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i), - &(parser.cf_opt_maps()->at(i)), sanity_check_level); + s = VerifyCFOptions(config_options, cf_opts[i], parser.cf_opts()->at(i), + &(parser.cf_opt_maps()->at(i))); if (!s.ok()) { return s; } - s = VerifyTableFactory(cf_opts[i].table_factory.get(), - parser.cf_opts()->at(i).table_factory.get(), - sanity_check_level); + s = VerifyTableFactory(config_options, cf_opts[i].table_factory.get(), + parser.cf_opts()->at(i).table_factory.get()); if (!s.ok()) { return s; } @@ -734,16 +755,16 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( } Status RocksDBOptionsParser::VerifyDBOptions( - const DBOptions& base_opt, const DBOptions& persisted_opt, - const std::unordered_map* /*opt_map*/, - OptionsSanityCheckLevel sanity_check_level) { + const ConfigOptions& config_options, const DBOptions& base_opt, + const DBOptions& persisted_opt, + const std::unordered_map* /*opt_map*/) { for (const auto& pair : db_options_type_info) { if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; } - if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (DBOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) { if (!AreEqualOptions(reinterpret_cast(&base_opt), reinterpret_cast(&persisted_opt), pair.second, pair.first, nullptr)) { @@ -771,17 +792,16 @@ Status RocksDBOptionsParser::VerifyDBOptions( } Status RocksDBOptionsParser::VerifyCFOptions( - const ColumnFamilyOptions& base_opt, + const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& persisted_opt, - const std::unordered_map* persisted_opt_map, - OptionsSanityCheckLevel sanity_check_level) { + const std::unordered_map* persisted_opt_map) { for (const auto& pair : cf_options_type_info) { if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; } - if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (CFOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) { if (!AreEqualOptions(reinterpret_cast(&base_opt), reinterpret_cast(&persisted_opt), pair.second, pair.first, persisted_opt_map)) { @@ -809,10 +829,10 @@ Status RocksDBOptionsParser::VerifyCFOptions( } Status RocksDBOptionsParser::VerifyTableFactory( - const TableFactory* base_tf, const TableFactory* file_tf, - OptionsSanityCheckLevel sanity_check_level) { + const ConfigOptions& config_options, const TableFactory* base_tf, + const TableFactory* file_tf) { if (base_tf && file_tf) { - if (sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone && + if (config_options.sanity_level > ConfigOptions::kSanityLevelNone && std::string(base_tf->Name()) != std::string(file_tf->Name())) { return Status::Corruption( "[RocksDBOptionsParser]: " @@ -820,11 +840,11 @@ Status RocksDBOptionsParser::VerifyTableFactory( } if (base_tf->Name() == BlockBasedTableFactory::kName) { return VerifyBlockBasedTableFactory( + config_options, static_cast_with_check(base_tf), static_cast_with_check(file_tf), - sanity_check_level); + const TableFactory>(file_tf)); } // TODO(yhchiang): add checks for other table factory types } else { diff --git a/options/options_parser.h b/options/options_parser.h index a88b806f8..2ebdec6d7 100644 --- a/options/options_parser.h +++ b/options/options_parser.h @@ -9,14 +9,15 @@ #include #include -#include "options/options_sanity_check.h" #include "rocksdb/env.h" #include "rocksdb/options.h" -#include "table/block_based/block_based_table_factory.h" namespace ROCKSDB_NAMESPACE { #ifndef ROCKSDB_LITE +struct ConfigOptions; +class OptionTypeInfo; +class TableFactory; #define ROCKSDB_OPTION_FILE_MAJOR 1 #define ROCKSDB_OPTION_FILE_MINOR 1 @@ -36,6 +37,11 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, const std::vector& cf_names, const std::vector& cf_opts, const std::string& file_name, FileSystem* fs); +Status PersistRocksDBOptions(const ConfigOptions& config_options, + const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs); extern bool AreEqualOptions( const char* opt1, const char* opt2, const OptionTypeInfo& type_info, @@ -52,6 +58,10 @@ class RocksDBOptionsParser { // If 0 is given, a default value will be used. Status Parse(const std::string& file_name, FileSystem* fs, bool ignore_unknown_options, size_t file_readahead_size); + + Status Parse(const ConfigOptions& config_options, + const std::string& file_name, FileSystem* fs); + static std::string TrimAndRemoveComment(const std::string& line, const bool trim_only = false); @@ -70,27 +80,26 @@ class RocksDBOptionsParser { return GetCFOptionsImpl(name); } size_t NumColumnFamilies() { return cf_opts_.size(); } - static Status VerifyRocksDBOptionsFromFile( - const DBOptions& db_opt, const std::vector& cf_names, + const ConfigOptions& config_options, const DBOptions& db_opt, + const std::vector& cf_names, const std::vector& cf_opts, - const std::string& file_name, FileSystem* fs, - OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch, - bool ignore_unknown_options = false); - + const std::string& file_name, FileSystem* fs); static Status VerifyDBOptions( - const DBOptions& base_opt, const DBOptions& new_opt, - const std::unordered_map* new_opt_map = nullptr, - OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); + const ConfigOptions& config_options, const DBOptions& base_opt, + const DBOptions& new_opt, + const std::unordered_map* new_opt_map = + nullptr); static Status VerifyCFOptions( - const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt, - const std::unordered_map* new_opt_map = nullptr, - OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); + const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, + const ColumnFamilyOptions& new_opt, + const std::unordered_map* new_opt_map = + nullptr); - static Status VerifyTableFactory( - const TableFactory* base_tf, const TableFactory* file_tf, - OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); + static Status VerifyTableFactory(const ConfigOptions& config_options, + const TableFactory* base_tf, + const TableFactory* file_tf); static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); @@ -106,10 +115,10 @@ class RocksDBOptionsParser { Status ParseStatement(std::string* name, std::string* value, const std::string& line, const int line_num); - Status EndSection(const OptionSection section, const std::string& title, - const std::string& section_arg, - const std::unordered_map& opt_map, - bool ignore_unknown_options); + Status EndSection( + const ConfigOptions& config_options, const OptionSection section, + const std::string& title, const std::string& section_arg, + const std::unordered_map& opt_map); Status ValidityCheck(); diff --git a/options/options_sanity_check.cc b/options/options_sanity_check.cc index 47cff31cf..f73555d6a 100644 --- a/options/options_sanity_check.cc +++ b/options/options_sanity_check.cc @@ -10,25 +10,26 @@ namespace ROCKSDB_NAMESPACE { namespace { -OptionsSanityCheckLevel SanityCheckLevelHelper( - const std::unordered_map& smap, +ConfigOptions::SanityLevel SanityCheckLevelHelper( + const std::unordered_map& smap, const std::string& name) { auto iter = smap.find(name); - return iter != smap.end() ? iter->second : kSanityLevelExactMatch; + return iter != smap.end() ? iter->second + : ConfigOptions::kSanityLevelExactMatch; } } -OptionsSanityCheckLevel DBOptionSanityCheckLevel( +ConfigOptions::SanityLevel DBOptionSanityCheckLevel( const std::string& option_name) { return SanityCheckLevelHelper(sanity_level_db_options, option_name); } -OptionsSanityCheckLevel CFOptionSanityCheckLevel( +ConfigOptions::SanityLevel CFOptionSanityCheckLevel( const std::string& option_name) { return SanityCheckLevelHelper(sanity_level_cf_options, option_name); } -OptionsSanityCheckLevel BBTOptionSanityCheckLevel( +ConfigOptions::SanityLevel BBTOptionSanityCheckLevel( const std::string& option_name) { return SanityCheckLevelHelper(sanity_level_bbt_options, option_name); } diff --git a/options/options_sanity_check.h b/options/options_sanity_check.h index ab8ec6d4d..64f8d9c50 100644 --- a/options/options_sanity_check.h +++ b/options/options_sanity_check.h @@ -8,41 +8,36 @@ #include #include +#include "rocksdb/convenience.h" #include "rocksdb/rocksdb_namespace.h" #ifndef ROCKSDB_LITE namespace ROCKSDB_NAMESPACE { // This enum defines the RocksDB options sanity level. -enum OptionsSanityCheckLevel : unsigned char { - // Performs no sanity check at all. - kSanityLevelNone = 0x00, - // Performs minimum check to ensure the RocksDB instance can be - // opened without corrupting / mis-interpreting the data. - kSanityLevelLooselyCompatible = 0x01, - // Perform exact match sanity check. - kSanityLevelExactMatch = 0xFF, -}; // The sanity check level for DB options -static const std::unordered_map - sanity_level_db_options {}; +static const std::unordered_map + sanity_level_db_options{}; // The sanity check level for column-family options -static const std::unordered_map +static const std::unordered_map sanity_level_cf_options = { - {"comparator", kSanityLevelLooselyCompatible}, - {"table_factory", kSanityLevelLooselyCompatible}, - {"merge_operator", kSanityLevelLooselyCompatible}}; + {"comparator", + ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible}, + {"table_factory", + ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible}, + {"merge_operator", + ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible}}; // The sanity check level for block-based table options -static const std::unordered_map - sanity_level_bbt_options {}; +static const std::unordered_map + sanity_level_bbt_options{}; -OptionsSanityCheckLevel DBOptionSanityCheckLevel( +ConfigOptions::SanityLevel DBOptionSanityCheckLevel( const std::string& options_name); -OptionsSanityCheckLevel CFOptionSanityCheckLevel( +ConfigOptions::SanityLevel CFOptionSanityCheckLevel( const std::string& options_name); -OptionsSanityCheckLevel BBTOptionSanityCheckLevel( +ConfigOptions::SanityLevel BBTOptionSanityCheckLevel( const std::string& options_name); } // namespace ROCKSDB_NAMESPACE diff --git a/options/options_test.cc b/options/options_test.cc index b795daa0e..78f39fd3c 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -143,6 +143,1241 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"strict_bytes_per_sync", "true"}, }; + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions exact, loose; + exact.input_strings_escaped = false; + exact.ignore_unknown_options = false; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + + loose.input_strings_escaped = false; + loose.ignore_unknown_options = true; + ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 1U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2); + ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3); + ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99); + ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999); + ASSERT_EQ(new_cf_opt.compression, kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U); + ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD); + ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression); + ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4); + ASSERT_EQ(new_cf_opt.compression_opts.level, 5); + ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u); + ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u); + ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 9u); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, 10u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.num_levels, 8); + ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8); + ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9); + ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10); + ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast(12)); + ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U); + ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18); + ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21); + ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211); + ASSERT_EQ(new_cf_opt.arena_block_size, 22U); + ASSERT_EQ(new_cf_opt.disable_auto_compactions, true); + ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel); + ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst); + ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size, + static_cast(23)); + ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations, + static_cast(24)); + ASSERT_EQ(new_cf_opt.inplace_update_support, true); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U); + ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26); + ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true); + ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U); + ASSERT_EQ(new_cf_opt.bloom_locality, 29U); + ASSERT_EQ(new_cf_opt.max_successive_merges, 30U); + ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr); + ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true); + ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()), + "rocksdb.FixedPrefix.31"); + + cf_options_map["write_buffer_size"] = "hello"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + cf_options_map["write_buffer_size"] = "1"; + ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + + cf_options_map["unknown_option"] = "1"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // ignore_unknown_options=true;input_strings_escaped=false + ASSERT_OK(GetColumnFamilyOptionsFromMap(loose, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(loose, base_cf_opt, new_cf_opt)); + ASSERT_NOK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + DBOptions base_db_opt; + DBOptions new_db_opt; + ASSERT_OK( + GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.create_missing_column_families, true); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.max_open_files, 32); + ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33)); + ASSERT_EQ(new_db_opt.use_fsync, true); + ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir"); + ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir"); + ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros, + static_cast(34)); + ASSERT_EQ(new_db_opt.max_background_compactions, 35); + ASSERT_EQ(new_db_opt.max_background_flushes, 36); + ASSERT_EQ(new_db_opt.max_log_file_size, 37U); + ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U); + ASSERT_EQ(new_db_opt.keep_log_file_num, 39U); + ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U); + ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast(40)); + ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41); + ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast(43)); + ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast(44)); + ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U); + ASSERT_EQ(new_db_opt.allow_mmap_reads, true); + ASSERT_EQ(new_db_opt.allow_mmap_writes, false); + ASSERT_EQ(new_db_opt.use_direct_reads, false); + ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false); + ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true); + ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false); + ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U); + ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U); + ASSERT_EQ(new_db_opt.persist_stats_to_disk, false); + ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U); + ASSERT_EQ(new_db_opt.advise_random_on_open, true); + ASSERT_EQ(new_db_opt.use_adaptive_mutex, false); + ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true); + ASSERT_EQ(new_db_opt.compaction_readahead_size, 100); + ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728); + ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159); + ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast(47)); + ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast(48)); + ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true); + + db_options_map["max_open_files"] = "hello"; + ASSERT_NOK( + GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); + + // unknow options should fail parsing without ignore_unknown_options = true + db_options_map["unknown_db_option"] = "1"; + ASSERT_NOK( + GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + + ASSERT_OK( + GetDBOptionsFromMap(loose, base_db_opt, db_options_map, &new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); + ASSERT_NOK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in + // ROCKSDB_LITE +TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + base_cf_opt.table_factory.reset(); + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, "", + &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=5", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 5U); + ASSERT_TRUE(new_cf_opt.table_factory == nullptr); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=6;", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 6U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, " write_buffer_size = 7 ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 7U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, " write_buffer_size = 8 ; ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 8U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 9U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=11; max_write_buffer_number = 12 ;", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 11U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12); + // Wrong name "max_write_buffer_number_" + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number_=14;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Comparator from object registry + std::string kCompName = "reverse_comp"; + ObjectLibrary::Default()->Register( + kCompName, + [](const std::string& /*name*/, + std::unique_ptr* /*guard*/, + std::string* /* errmsg */) { return ReverseBytewiseComparator(); }); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "comparator=" + kCompName + ";", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator()); + + // MergeOperator from object registry + std::unique_ptr bxo(new BytesXOROperator()); + std::string kMoName = bxo->Name(); + ObjectLibrary::Default()->Register( + kMoName, + [](const std::string& /*name*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new BytesXOROperator()); + return guard->get(); + }); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "merge_operator=" + kMoName + ";", + &new_cf_opt)); + ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name())); + + // Wrong key/value pair + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Error Paring value + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Missing option name + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=13; =100;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + const uint64_t kilo = 1024UL; + const uint64_t mega = 1024 * kilo; + const uint64_t giga = 1024 * mega; + const uint64_t tera = 1024 * giga; + + // Units (k) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo); + // Units (m) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "max_write_buffer_number=16m;inplace_update_num_locks=17M", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega); + // Units (g) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=18g;prefix_extractor=capped:8;" + "arena_block_size=19G", + &new_cf_opt)); + + ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga); + ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga); + ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr); + std::string prefix_name(new_cf_opt.prefix_extractor->Name()); + ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8"); + + // Units (t) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=20t;arena_block_size=21T", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera); + ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera); + + // Nested block based table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Last one + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;}", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Mismatch curly braces + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={{{block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Unexpected chars after closing curly brace + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa;" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Invalid block based table option + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={xx_block_size=4;}", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=true", + &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=false", + &new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=junk", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Nested plain table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + + // memtable factory + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "memtable=skip_list:10;arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); +} + +TEST_F(OptionsTest, OldInterfaceTest) { + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions exact; + + ASSERT_OK(GetColumnFamilyOptionsFromString( + base_cf_opt, + "write_buffer_size=18;prefix_extractor=capped:8;" + "arena_block_size=19", + &new_cf_opt)); + + ASSERT_EQ(new_cf_opt.write_buffer_size, 18); + ASSERT_EQ(new_cf_opt.arena_block_size, 19); + ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr); + + // And with a bad option + ASSERT_NOK(GetColumnFamilyOptionsFromString( + base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={xx_block_size=4;}", + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + std::unordered_map cf_options_map = { + {"write_buffer_size", "1"}, + {"max_write_buffer_number", "2"}, + {"min_write_buffer_number_to_merge", "3"}, + }; + ASSERT_OK( + GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, &new_cf_opt)); + cf_options_map["unknown_option"] = "1"; + ASSERT_NOK( + GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, + &new_cf_opt, true, true)); + + DBOptions base_db_opt; + DBOptions new_db_opt; + std::unordered_map db_options_map = { + {"create_if_missing", "false"}, + {"create_missing_column_families", "true"}, + {"error_if_exists", "false"}, + {"paranoid_checks", "true"}, + {"max_open_files", "32"}, + }; + ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.create_missing_column_families, true); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.max_open_files, 32); + db_options_map["unknown_option"] = "1"; + ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt, true, + true)); + ASSERT_OK(GetDBOptionsFromString( + base_db_opt, + "create_if_missing=false;error_if_exists=false;max_open_files=42;", + &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.max_open_files, 42); + ASSERT_NOK(GetDBOptionsFromString( + base_db_opt, + "create_if_missing=false;error_if_exists=false;max_open_files=42;" + "unknown_option=1;", + &new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); +} + +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported +TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { + BlockBasedTableOptions table_opt; + BlockBasedTableOptions new_opt; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + // make sure default values are overwritten by something else + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kHashSearch;" + "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" + "block_cache=1M;block_cache_compressed=1k;block_size=1024;" + "block_size_deviation=8;block_restart_interval=4;" + "format_version=5;whole_key_filtering=1;" + "filter_policy=bloomfilter:4.567:false;", + &new_opt)); + ASSERT_TRUE(new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch); + ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash); + ASSERT_TRUE(new_opt.hash_index_allow_collision); + ASSERT_TRUE(new_opt.no_block_cache); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL); + ASSERT_EQ(new_opt.block_size, 1024UL); + ASSERT_EQ(new_opt.block_size_deviation, 8); + ASSERT_EQ(new_opt.block_restart_interval, 4); + ASSERT_EQ(new_opt.format_version, 5U); + ASSERT_EQ(new_opt.whole_key_filtering, true); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + const BloomFilterPolicy& bfp = + dynamic_cast(*new_opt.filter_policy); + EXPECT_EQ(bfp.GetMillibitsPerKey(), 4567); + EXPECT_EQ(bfp.GetWholeBitsPerKey(), 5); + + // unknown option + ASSERT_NOK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearch;" + "bad_option=1", + &new_opt)); + ASSERT_EQ(static_cast(table_opt.cache_index_and_filter_blocks), + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized index type + ASSERT_NOK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX", &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized checksum type + ASSERT_NOK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;checksum=kxxHashXX", &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized filter policy name + ASSERT_NOK( + GetBlockBasedTableOptionsFromString(config_options, table_opt, + "cache_index_and_filter_blocks=1;" + "filter_policy=bloomfilterxx:4:true", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy); + + // unrecognized filter policy config + ASSERT_NOK( + GetBlockBasedTableOptionsFromString(config_options, table_opt, + "cache_index_and_filter_blocks=1;" + "filter_policy=bloomfilter:4", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy); + + // Check block cache options are overwritten when specified + // in new format as a struct. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache_compressed)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache_compressed)->GetHighPriPoolRatio(), + 0.5); + + // Set only block cache capacity. Check other values are + // reset to default values. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=2M};" + "block_cache_compressed={capacity=2M}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL); + // Default values + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetNumShardBits(), + GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity())); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetHighPriPoolRatio(), + 0.5); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL); + // Default values + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache_compressed)->GetNumShardBits(), + GetDefaultCacheShardBits( + new_opt.block_cache_compressed->GetCapacity())); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache_compressed) + ->GetHighPriPoolRatio(), + 0.5); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={num_shard_bits=5;" + "high_pri_pool_ratio=0.0;}", + &new_opt)); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetNumShardBits(), 5); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache_compressed)->GetNumShardBits(), 5); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache_compressed) + ->GetHighPriPoolRatio(), + 0.0); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetHighPriPoolRatio(), + 0.5); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache_compressed)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache_compressed) + ->GetHighPriPoolRatio(), + 0.5); +} +#endif // !ROCKSDB_LITE + + +#ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported +TEST_F(OptionsTest, GetPlainTableOptionsFromString) { + PlainTableOptions table_opt; + PlainTableOptions new_opt; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + // make sure default values are overwritten by something else + ASSERT_OK(GetPlainTableOptionsFromString( + config_options, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;" + "full_scan_mode=true;store_index_in_file=true", + &new_opt)); + ASSERT_EQ(new_opt.user_key_len, 66u); + ASSERT_EQ(new_opt.bloom_bits_per_key, 20); + ASSERT_EQ(new_opt.hash_table_ratio, 0.5); + ASSERT_EQ(new_opt.index_sparseness, 8); + ASSERT_EQ(new_opt.huge_page_tlb_size, 4); + ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix); + ASSERT_TRUE(new_opt.full_scan_mode); + ASSERT_TRUE(new_opt.store_index_in_file); + + // unknown option + ASSERT_NOK(GetPlainTableOptionsFromString( + config_options, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "bad_option=1", + &new_opt)); + + // unrecognized EncodingType + ASSERT_NOK(GetPlainTableOptionsFromString( + config_options, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "encoding_type=kPrefixXX", + &new_opt)); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported +TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { + std::unique_ptr new_mem_factory = nullptr; + + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000", + &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist", + &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000", + &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt", + &new_mem_factory)); + + ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory)); + // CuckooHash memtable is already removed. + ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory)); + + ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory)); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite +TEST_F(OptionsTest, GetOptionsFromStringTest) { + Options base_options, new_options; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + base_options.write_buffer_size = 20; + base_options.min_write_buffer_number_to_merge = 15; + BlockBasedTableOptions block_based_table_options; + block_based_table_options.cache_index_and_filter_blocks = true; + base_options.table_factory.reset( + NewBlockBasedTableFactory(block_based_table_options)); + + // Register an Env with object registry. + const static char* kCustomEnvName = "CustomEnv"; + class CustomEnv : public EnvWrapper { + public: + explicit CustomEnv(Env* _target) : EnvWrapper(_target) {} + }; + + ObjectLibrary::Default()->Register( + kCustomEnvName, + [](const std::string& /*name*/, std::unique_ptr* /*env_guard*/, + std::string* /* errmsg */) { + static CustomEnv env(Env::Default()); + return &env; + }); + + ASSERT_OK(GetOptionsFromString( + config_options, base_options, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;" + "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files=" + "1;" + "rate_limiter_bytes_per_sec=1024;env=CustomEnv", + &new_options)); + + ASSERT_EQ(new_options.compression_opts.window_bits, 4); + ASSERT_EQ(new_options.compression_opts.level, 5); + ASSERT_EQ(new_options.compression_opts.strategy, 6); + ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u); + ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u); + ASSERT_EQ(new_options.compression_opts.parallel_threads, 1u); + ASSERT_EQ(new_options.compression_opts.enabled, false); + ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption); + ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_options.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u); + ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u); + ASSERT_EQ(new_options.bottommost_compression_opts.parallel_threads, 1u); + ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); + ASSERT_EQ(new_options.write_buffer_size, 10U); + ASSERT_EQ(new_options.max_write_buffer_number, 16); + BlockBasedTableOptions new_block_based_table_options = + dynamic_cast(new_options.table_factory.get()) + ->table_options(); + ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20); + ASSERT_EQ(new_block_based_table_options.block_size, 4U); + // don't overwrite block based table options + ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks); + + ASSERT_EQ(new_options.create_if_missing, true); + ASSERT_EQ(new_options.max_open_files, 1); + ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); + Env* newEnv = new_options.env; + ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv)); + ASSERT_EQ(newEnv, new_options.env); + + // Test the old interfaxe + ASSERT_OK(GetOptionsFromString( + base_options, + "write_buffer_size=22;max_write_buffer_number=33;max_open_files=44;", + &new_options)); + ASSERT_EQ(new_options.write_buffer_size, 22U); + ASSERT_EQ(new_options.max_write_buffer_number, 33); + ASSERT_EQ(new_options.max_open_files, 44); +} + +TEST_F(OptionsTest, DBOptionsSerialization) { + Options base_options, new_options; + Random rnd(301); + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + // Phase 1: Make big change in base_options + test::RandomInitDBOptions(&base_options, &rnd); + + // Phase 2: obtain a string from base_option + std::string base_options_file_content; + ASSERT_OK(GetStringFromDBOptions(config_options, base_options, + &base_options_file_content)); + + // Phase 3: Set new_options from the derived string and expect + // new_options == base_options + ASSERT_OK(GetDBOptionsFromString(config_options, DBOptions(), + base_options_file_content, &new_options)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_options, + new_options)); +} + +TEST_F(OptionsTest, OptionsComposeDecompose) { + // build an Options from DBOptions + CFOptions, then decompose it to verify + // we get same constituent options. + DBOptions base_db_opts; + ColumnFamilyOptions base_cf_opts; + ConfigOptions + config_options; // Use default for ignore(false) and check (exact) + config_options.input_strings_escaped = false; + + Random rnd(301); + test::RandomInitDBOptions(&base_db_opts, &rnd); + test::RandomInitCFOptions(&base_cf_opts, base_db_opts, &rnd); + + Options base_opts(base_db_opts, base_cf_opts); + DBOptions new_db_opts(base_opts); + ColumnFamilyOptions new_cf_opts(base_opts); + + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_db_opts, + new_db_opts)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opts, + new_cf_opts)); + delete new_cf_opts.compaction_filter; +} + +TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { + Options options; + ColumnFamilyOptions base_opt, new_opt; + Random rnd(302); + ConfigOptions config_options; + config_options.input_strings_escaped = false; + + // Phase 1: randomly assign base_opt + // custom type options + test::RandomInitCFOptions(&base_opt, options, &rnd); + + // Phase 2: obtain a string from base_opt + std::string base_options_file_content; + ASSERT_OK(GetStringFromColumnFamilyOptions(config_options, base_opt, + &base_options_file_content)); + + // Phase 3: Set new_opt from the derived string and expect + // new_opt == base_opt + ASSERT_OK( + GetColumnFamilyOptionsFromString(config_options, ColumnFamilyOptions(), + base_options_file_content, &new_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt)); + if (base_opt.compaction_filter) { + delete base_opt.compaction_filter; + } +} + +#endif // !ROCKSDB_LITE + +Status StringToMap( + const std::string& opts_str, + std::unordered_map* opts_map); + +#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE +TEST_F(OptionsTest, StringToMapTest) { + std::unordered_map opts_map; + // Regular options + ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "v2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Value with '=' + opts_map.clear(); + ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map)); + ASSERT_EQ(opts_map["k1"], "=v1"); + ASSERT_EQ(opts_map["k2"], "v2="); + // Overwrriten option + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Empty value + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_EQ(opts_map["k3"], "v3"); + ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); + ASSERT_EQ(opts_map["k4"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_EQ(opts_map["k3"], "v3"); + ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); + ASSERT_EQ(opts_map["k4"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); + ASSERT_EQ(opts_map["k3"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); + ASSERT_EQ(opts_map["k3"], ""); + // Regular nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Multi-level nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};" + "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}"); + ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}"); + ASSERT_EQ(opts_map["k4"], "v4"); + // Garbage inside curly braces + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "dfad="); + ASSERT_EQ(opts_map["k3"], "="); + ASSERT_EQ(opts_map["k4"], "v4"); + // Empty nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}"); + // With random spaces + opts_map.clear(); + ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; " + "k3={ { } }; k4= v4 ", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}"); + ASSERT_EQ(opts_map["k3"], "{ }"); + ASSERT_EQ(opts_map["k4"], "v4"); + + // Empty key + ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map)); + ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map)); + // Mismatch curly braces + ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map)); + // However this is valid! + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "}"); + ASSERT_EQ(opts_map["k3"], "v3"); + + // Invalid chars after closing curly brace + ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map)); +} +#endif // ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE +TEST_F(OptionsTest, StringToMapRandomTest) { + std::unordered_map opts_map; + // Make sure segfault is not hit by semi-random strings + + std::vector bases = { + "a={aa={};tt={xxx={}}};c=defff", + "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}", + "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"}; + + for (std::string base : bases) { + for (int rand_seed = 301; rand_seed < 401; rand_seed++) { + Random rnd(rand_seed); + for (int attempt = 0; attempt < 10; attempt++) { + std::string str = base; + // Replace random position to space + size_t pos = static_cast( + rnd.Uniform(static_cast(base.size()))); + str[pos] = ' '; + Status s = StringToMap(str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + opts_map.clear(); + } + } + } + + // Random Construct a string + std::vector chars = {'{', '}', ' ', '=', ';', 'c'}; + for (int rand_seed = 301; rand_seed < 1301; rand_seed++) { + Random rnd(rand_seed); + int len = rnd.Uniform(30); + std::string str = ""; + for (int attempt = 0; attempt < len; attempt++) { + // Add a random character + size_t pos = static_cast( + rnd.Uniform(static_cast(chars.size()))); + str.append(1, chars[pos]); + } + Status s = StringToMap(str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + s = StringToMap("name=" + str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + opts_map.clear(); + } +} + +TEST_F(OptionsTest, GetStringFromCompressionType) { + std::string res; + + ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression)); + ASSERT_EQ(res, "kNoCompression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression)); + ASSERT_EQ(res, "kSnappyCompression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption)); + ASSERT_EQ(res, "kDisableCompressionOption"); + + ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression)); + ASSERT_EQ(res, "kLZ4Compression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression)); + ASSERT_EQ(res, "kZlibCompression"); + + ASSERT_NOK( + GetStringFromCompressionType(&res, static_cast(-10))); +} +#endif // !ROCKSDB_LITE + +TEST_F(OptionsTest, ConvertOptionsTest) { + LevelDBOptions leveldb_opt; + Options converted_opt = ConvertOptions(leveldb_opt); + + ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing); + ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists); + ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks); + ASSERT_EQ(converted_opt.env, leveldb_opt.env); + ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log); + ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size); + ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); + ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); + + std::shared_ptr tb_guard = converted_opt.table_factory; + BlockBasedTableFactory* table_factory = + dynamic_cast(converted_opt.table_factory.get()); + + ASSERT_TRUE(table_factory != nullptr); + + const BlockBasedTableOptions table_opt = table_factory->table_options(); + + ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20); + ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size); + ASSERT_EQ(table_opt.block_restart_interval, + leveldb_opt.block_restart_interval); + ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy); +} + +#ifndef ROCKSDB_LITE +// This test suite tests the old APIs into the Configure options methods. +// Once those APIs are officially deprecated, this test suite can be deleted. +class OptionsOldApiTest : public testing::Test {}; + +TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { + std::unordered_map cf_options_map = { + {"write_buffer_size", "1"}, + {"max_write_buffer_number", "2"}, + {"min_write_buffer_number_to_merge", "3"}, + {"max_write_buffer_number_to_maintain", "99"}, + {"max_write_buffer_size_to_maintain", "-99999"}, + {"compression", "kSnappyCompression"}, + {"compression_per_level", + "kNoCompression:" + "kSnappyCompression:" + "kZlibCompression:" + "kBZip2Compression:" + "kLZ4Compression:" + "kLZ4HCCompression:" + "kXpressCompression:" + "kZSTD:" + "kZSTDNotFinalCompression"}, + {"bottommost_compression", "kLZ4Compression"}, + {"bottommost_compression_opts", "5:6:7:8:9:10:true"}, + {"compression_opts", "4:5:6:7:8:9:true"}, + {"num_levels", "8"}, + {"level0_file_num_compaction_trigger", "8"}, + {"level0_slowdown_writes_trigger", "9"}, + {"level0_stop_writes_trigger", "10"}, + {"target_file_size_base", "12"}, + {"target_file_size_multiplier", "13"}, + {"max_bytes_for_level_base", "14"}, + {"level_compaction_dynamic_level_bytes", "true"}, + {"max_bytes_for_level_multiplier", "15.0"}, + {"max_bytes_for_level_multiplier_additional", "16:17:18"}, + {"max_compaction_bytes", "21"}, + {"soft_rate_limit", "1.1"}, + {"hard_rate_limit", "2.1"}, + {"hard_pending_compaction_bytes_limit", "211"}, + {"arena_block_size", "22"}, + {"disable_auto_compactions", "true"}, + {"compaction_style", "kCompactionStyleLevel"}, + {"compaction_pri", "kOldestSmallestSeqFirst"}, + {"verify_checksums_in_compaction", "false"}, + {"compaction_options_fifo", "23"}, + {"max_sequential_skip_in_iterations", "24"}, + {"inplace_update_support", "true"}, + {"report_bg_io_stats", "true"}, + {"compaction_measure_io_stats", "false"}, + {"inplace_update_num_locks", "25"}, + {"memtable_prefix_bloom_size_ratio", "0.26"}, + {"memtable_whole_key_filtering", "true"}, + {"memtable_huge_page_size", "28"}, + {"bloom_locality", "29"}, + {"max_successive_merges", "30"}, + {"min_partial_merge_operands", "31"}, + {"prefix_extractor", "fixed:31"}, + {"optimize_filters_for_hits", "true"}, + }; + + std::unordered_map db_options_map = { + {"create_if_missing", "false"}, + {"create_missing_column_families", "true"}, + {"error_if_exists", "false"}, + {"paranoid_checks", "true"}, + {"max_open_files", "32"}, + {"max_total_wal_size", "33"}, + {"use_fsync", "true"}, + {"db_log_dir", "/db_log_dir"}, + {"wal_dir", "/wal_dir"}, + {"delete_obsolete_files_period_micros", "34"}, + {"max_background_compactions", "35"}, + {"max_background_flushes", "36"}, + {"max_log_file_size", "37"}, + {"log_file_time_to_roll", "38"}, + {"keep_log_file_num", "39"}, + {"recycle_log_file_num", "5"}, + {"max_manifest_file_size", "40"}, + {"table_cache_numshardbits", "41"}, + {"WAL_ttl_seconds", "43"}, + {"WAL_size_limit_MB", "44"}, + {"manifest_preallocation_size", "45"}, + {"allow_mmap_reads", "true"}, + {"allow_mmap_writes", "false"}, + {"use_direct_reads", "false"}, + {"use_direct_io_for_flush_and_compaction", "false"}, + {"is_fd_close_on_exec", "true"}, + {"skip_log_error_on_recovery", "false"}, + {"stats_dump_period_sec", "46"}, + {"stats_persist_period_sec", "57"}, + {"persist_stats_to_disk", "false"}, + {"stats_history_buffer_size", "69"}, + {"advise_random_on_open", "true"}, + {"use_adaptive_mutex", "false"}, + {"new_table_reader_for_compaction_inputs", "true"}, + {"compaction_readahead_size", "100"}, + {"random_access_max_buffer_size", "3145728"}, + {"writable_file_max_buffer_size", "314159"}, + {"bytes_per_sync", "47"}, + {"wal_bytes_per_sync", "48"}, + {"strict_bytes_per_sync", "true"}, + }; + ColumnFamilyOptions base_cf_opt; ColumnFamilyOptions new_cf_opt; ASSERT_OK(GetColumnFamilyOptionsFromMap( @@ -216,7 +1451,11 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { cf_options_map["write_buffer_size"] = "hello"; ASSERT_NOK(GetColumnFamilyOptionsFromMap( base_cf_opt, cf_options_map, &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ConfigOptions exact, loose; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); cf_options_map["write_buffer_size"] = "1"; ASSERT_OK(GetColumnFamilyOptionsFromMap( @@ -225,18 +1464,16 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { cf_options_map["unknown_option"] = "1"; ASSERT_NOK(GetColumnFamilyOptionsFromMap( base_cf_opt, cf_options_map, &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, &new_cf_opt, false, /* input_strings_escaped */ true /* ignore_unknown_options */)); ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( - base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */ - kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/)); + loose, base_cf_opt, new_cf_opt, nullptr /* new_opt_map */)); ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( - base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */ - kSanityLevelExactMatch /* default for VerifyCFOptions */)); + exact /* default for VerifyCFOptions */, base_cf_opt, new_cf_opt, nullptr)); DBOptions base_db_opt; DBOptions new_db_opt; @@ -285,31 +1522,22 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { db_options_map["max_open_files"] = "hello"; ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( - base_db_opt, new_db_opt, nullptr, /* new_opt_map */ - kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); // unknow options should fail parsing without ignore_unknown_options = true db_options_map["unknown_db_option"] = "1"; ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt, false, /* input_strings_escaped */ true /* ignore_unknown_options */)); - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( - base_db_opt, new_db_opt, nullptr, /* new_opt_map */ - kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */)); - ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions( - base_db_opt, new_db_opt, nullptr, /* new_opt_mat */ - kSanityLevelExactMatch /* default for VerifyDBOptions */)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); + ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); } -#endif // !ROCKSDB_LITE -#ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in - // ROCKSDB_LITE -TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { +TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) { ColumnFamilyOptions base_cf_opt; ColumnFamilyOptions new_cf_opt; base_cf_opt.table_factory.reset(); @@ -340,7 +1568,9 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "write_buffer_size=13;max_write_buffer_number_=14;", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ConfigOptions exact; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); // Comparator from object registry std::string kCompName = "reverse_comp"; @@ -372,17 +1602,17 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { // Wrong key/value pair ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); // Error Paring value ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); // Missing option name ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "write_buffer_size=13; =100;", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); const uint64_t kilo = 1024UL; const uint64_t mega = 1024 * kilo; @@ -444,7 +1674,7 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { "block_based_table_factory={{{block_size=4;};" "arena_block_size=1024", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); // Unexpected chars after closing curly brace ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, @@ -452,27 +1682,27 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { "block_based_table_factory={block_size=4;}};" "arena_block_size=1024", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "write_buffer_size=10;max_write_buffer_number=16;" "block_based_table_factory={block_size=4;}xdfa;" "arena_block_size=1024", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "write_buffer_size=10;max_write_buffer_number=16;" "block_based_table_factory={block_size=4;}xdfa", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); // Invalid block based table option ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "write_buffer_size=10;max_write_buffer_number=16;" "block_based_table_factory={xx_block_size=4;}", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "optimize_filters_for_hits=true", @@ -484,7 +1714,7 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "optimize_filters_for_hits=junk", &new_cf_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); // Nested plain table options // Empty @@ -511,10 +1741,8 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); } -#endif // !ROCKSDB_LITE - -#ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported -TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { + +TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) { BlockBasedTableOptions table_opt; BlockBasedTableOptions new_opt; // make sure default values are overwritten by something else @@ -688,11 +1916,8 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { ->GetHighPriPoolRatio(), 0.5); } -#endif // !ROCKSDB_LITE - - -#ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported -TEST_F(OptionsTest, GetPlainTableOptionsFromString) { + +TEST_F(OptionsOldApiTest, GetPlainTableOptionsFromString) { PlainTableOptions table_opt; PlainTableOptions new_opt; // make sure default values are overwritten by something else @@ -718,53 +1943,12 @@ TEST_F(OptionsTest, GetPlainTableOptionsFromString) { // unrecognized EncodingType ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, - "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" - "encoding_type=kPrefixXX", - &new_opt)); -} -#endif // !ROCKSDB_LITE - -#ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported -TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { - std::unique_ptr new_mem_factory = nullptr; - - ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory)); - ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory)); - ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory"); - ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt", - &new_mem_factory)); - - ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory)); - ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000", - &new_mem_factory)); - ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory"); - ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt", - &new_mem_factory)); - - ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist", - &new_mem_factory)); - ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000", - &new_mem_factory)); - ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory"); - ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt", - &new_mem_factory)); - - ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory)); - ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory)); - ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory"); - ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt", - &new_mem_factory)); - - ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory)); - // CuckooHash memtable is already removed. - ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory)); - - ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory)); + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "encoding_type=kPrefixXX", + &new_opt)); } -#endif // !ROCKSDB_LITE - -#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite -TEST_F(OptionsTest, GetOptionsFromStringTest) { + +TEST_F(OptionsOldApiTest, GetOptionsFromStringTest) { Options base_options, new_options; base_options.write_buffer_size = 20; base_options.min_write_buffer_number_to_merge = 15; @@ -831,7 +2015,7 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) { ASSERT_EQ(newEnv, new_options.env); } -TEST_F(OptionsTest, DBOptionsSerialization) { +TEST_F(OptionsOldApiTest, DBOptionsSerialization) { Options base_options, new_options; Random rnd(301); @@ -846,29 +2030,11 @@ TEST_F(OptionsTest, DBOptionsSerialization) { // new_options == base_options ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content, &new_options)); - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options)); -} - -TEST_F(OptionsTest, OptionsComposeDecompose) { - // build an Options from DBOptions + CFOptions, then decompose it to verify - // we get same constituent options. - DBOptions base_db_opts; - ColumnFamilyOptions base_cf_opts; - - Random rnd(301); - test::RandomInitDBOptions(&base_db_opts, &rnd); - test::RandomInitCFOptions(&base_cf_opts, base_db_opts, &rnd); - - Options base_opts(base_db_opts, base_cf_opts); - DBOptions new_db_opts(base_opts); - ColumnFamilyOptions new_cf_opts(base_opts); - - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opts, new_db_opts)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opts, new_cf_opts)); - delete new_cf_opts.compaction_filter; + ConfigOptions config_options; + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_options, new_options)); } -TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { +TEST_F(OptionsOldApiTest, ColumnFamilyOptionsSerialization) { Options options; ColumnFamilyOptions base_opt, new_opt; Random rnd(302); @@ -885,234 +2051,14 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { // new_opt == base_opt ASSERT_OK(GetColumnFamilyOptionsFromString( ColumnFamilyOptions(), base_options_file_content, &new_opt)); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt)); + ConfigOptions config_options; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt)); if (base_opt.compaction_filter) { delete base_opt.compaction_filter; } } - -#endif // !ROCKSDB_LITE - -Status StringToMap( - const std::string& opts_str, - std::unordered_map* opts_map); - -#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE -TEST_F(OptionsTest, StringToMapTest) { - std::unordered_map opts_map; - // Regular options - ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], "v2"); - ASSERT_EQ(opts_map["k3"], "v3"); - // Value with '=' - opts_map.clear(); - ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map)); - ASSERT_EQ(opts_map["k1"], "=v1"); - ASSERT_EQ(opts_map["k2"], "v2="); - // Overwrriten option - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v2"); - ASSERT_EQ(opts_map["k3"], "v3"); - // Empty value - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); - ASSERT_EQ(opts_map["k2"], ""); - ASSERT_EQ(opts_map["k3"], "v3"); - ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); - ASSERT_EQ(opts_map["k4"], ""); - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); - ASSERT_EQ(opts_map["k2"], ""); - ASSERT_EQ(opts_map["k3"], "v3"); - ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); - ASSERT_EQ(opts_map["k4"], ""); - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); - ASSERT_EQ(opts_map["k2"], ""); - ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); - ASSERT_EQ(opts_map["k3"], ""); - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); - ASSERT_EQ(opts_map["k2"], ""); - ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); - ASSERT_EQ(opts_map["k3"], ""); - // Regular nested options - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2"); - ASSERT_EQ(opts_map["k3"], "v3"); - // Multi-level nested options - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};" - "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4", - &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}"); - ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}"); - ASSERT_EQ(opts_map["k4"], "v4"); - // Garbage inside curly braces - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4", - &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], "dfad="); - ASSERT_EQ(opts_map["k3"], "="); - ASSERT_EQ(opts_map["k4"], "v4"); - // Empty nested options - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], ""); - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}"); - // With random spaces - opts_map.clear(); - ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; " - "k3={ { } }; k4= v4 ", - &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}"); - ASSERT_EQ(opts_map["k3"], "{ }"); - ASSERT_EQ(opts_map["k4"], "v4"); - - // Empty key - ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map)); - ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map)); - // Mismatch curly braces - ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map)); - // However this is valid! - opts_map.clear(); - ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map)); - ASSERT_EQ(opts_map["k1"], "v1"); - ASSERT_EQ(opts_map["k2"], "}"); - ASSERT_EQ(opts_map["k3"], "v3"); - - // Invalid chars after closing curly brace - ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map)); - ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map)); -} -#endif // ROCKSDB_LITE - -#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE -TEST_F(OptionsTest, StringToMapRandomTest) { - std::unordered_map opts_map; - // Make sure segfault is not hit by semi-random strings - - std::vector bases = { - "a={aa={};tt={xxx={}}};c=defff", - "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}", - "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"}; - - for (std::string base : bases) { - for (int rand_seed = 301; rand_seed < 401; rand_seed++) { - Random rnd(rand_seed); - for (int attempt = 0; attempt < 10; attempt++) { - std::string str = base; - // Replace random position to space - size_t pos = static_cast( - rnd.Uniform(static_cast(base.size()))); - str[pos] = ' '; - Status s = StringToMap(str, &opts_map); - ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); - opts_map.clear(); - } - } - } - - // Random Construct a string - std::vector chars = {'{', '}', ' ', '=', ';', 'c'}; - for (int rand_seed = 301; rand_seed < 1301; rand_seed++) { - Random rnd(rand_seed); - int len = rnd.Uniform(30); - std::string str = ""; - for (int attempt = 0; attempt < len; attempt++) { - // Add a random character - size_t pos = static_cast( - rnd.Uniform(static_cast(chars.size()))); - str.append(1, chars[pos]); - } - Status s = StringToMap(str, &opts_map); - ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); - s = StringToMap("name=" + str, &opts_map); - ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); - opts_map.clear(); - } -} - -TEST_F(OptionsTest, GetStringFromCompressionType) { - std::string res; - - ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression)); - ASSERT_EQ(res, "kNoCompression"); - - ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression)); - ASSERT_EQ(res, "kSnappyCompression"); - - ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption)); - ASSERT_EQ(res, "kDisableCompressionOption"); - - ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression)); - ASSERT_EQ(res, "kLZ4Compression"); - - ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression)); - ASSERT_EQ(res, "kZlibCompression"); - - ASSERT_NOK( - GetStringFromCompressionType(&res, static_cast(-10))); -} #endif // !ROCKSDB_LITE -TEST_F(OptionsTest, ConvertOptionsTest) { - LevelDBOptions leveldb_opt; - Options converted_opt = ConvertOptions(leveldb_opt); - - ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing); - ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists); - ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks); - ASSERT_EQ(converted_opt.env, leveldb_opt.env); - ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log); - ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size); - ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); - ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); - - std::shared_ptr tb_guard = converted_opt.table_factory; - BlockBasedTableFactory* table_factory = - dynamic_cast(converted_opt.table_factory.get()); - - ASSERT_TRUE(table_factory != nullptr); - - const BlockBasedTableOptions table_opt = table_factory->table_options(); - - ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20); - ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size); - ASSERT_EQ(table_opt.block_restart_interval, - leveldb_opt.block_restart_interval); - ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy); -} - #ifndef ROCKSDB_LITE class OptionsParserTest : public testing::Test { public: @@ -1157,10 +2103,14 @@ TEST_F(OptionsParserTest, Comment) { ASSERT_OK( parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt)); + ConfigOptions exact; + exact.input_strings_escaped = false; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, *parser.db_opt(), db_opt)); ASSERT_EQ(parser.NumColumnFamilies(), 1U); ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( - *parser.GetCFOptions("default"), cf_opt)); + exact, *parser.GetCFOptions("default"), cf_opt)); } TEST_F(OptionsParserTest, ExtraSpace) { @@ -1201,6 +2151,7 @@ TEST_F(OptionsParserTest, MissingDBOptions) { RocksDBOptionsParser parser; ASSERT_NOK( parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); + ; } TEST_F(OptionsParserTest, DoubleDBOptions) { @@ -1448,8 +2399,10 @@ void VerifyCFPointerTypedOptions( ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt, const std::unordered_map* new_cf_opt_map) { std::string name_buffer; - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, - new_cf_opt_map)); + ConfigOptions config_options; + config_options.input_strings_escaped = false; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, *base_cf_opt, + *new_cf_opt, new_cf_opt_map)); // change the name of merge operator back-and-forth { @@ -1460,11 +2413,11 @@ void VerifyCFPointerTypedOptions( // change the name and expect non-ok status merge_operator->SetName("some-other-name"); ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( - *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); // change the name back and expect ok status merge_operator->SetName(name_buffer); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, - new_cf_opt_map)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); } } @@ -1478,11 +2431,11 @@ void VerifyCFPointerTypedOptions( // change the name and expect non-ok status compaction_filter_factory->SetName("some-other-name"); ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( - *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); // change the name back and expect ok status compaction_filter_factory->SetName(name_buffer); - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, - new_cf_opt_map)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); } } @@ -1493,11 +2446,11 @@ void VerifyCFPointerTypedOptions( base_cf_opt->compaction_filter = nullptr; // set compaction_filter to nullptr and expect non-ok status ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( - *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); // set the value back and expect ok status base_cf_opt->compaction_filter = tmp_compaction_filter; - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, - new_cf_opt_map)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); } } @@ -1508,11 +2461,11 @@ void VerifyCFPointerTypedOptions( base_cf_opt->table_factory.reset(); // set table_factory to nullptr and expect non-ok status ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( - *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); // set the value back and expect ok status base_cf_opt->table_factory = tmp_table_factory; - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, - new_cf_opt_map)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); } } @@ -1523,11 +2476,11 @@ void VerifyCFPointerTypedOptions( base_cf_opt->memtable_factory.reset(); // set memtable_factory to nullptr and expect non-ok status ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( - *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); // set the value back and expect ok status base_cf_opt->memtable_factory = tmp_memtable_factory; - ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, - new_cf_opt_map)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); } } } @@ -1611,12 +2564,13 @@ TEST_F(OptionsParserTest, DumpAndParse) { } const std::string kOptionsFileName = "test-persisted-options.ini"; + // Use default for escaped(true), unknown(false) and check (exact) + ConfigOptions config_options; ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get())); RocksDBOptionsParser parser; - ASSERT_OK( - parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */)); + ASSERT_OK(parser.Parse(config_options, kOptionsFileName, fs_.get())); // Make sure block-based table factory options was deserialized correctly std::shared_ptr ttf = (*parser.cf_opts())[4].table_factory; @@ -1628,15 +2582,17 @@ TEST_F(OptionsParserTest, DumpAndParse) { parsed_bbto.cache_index_and_filter_blocks); ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( - base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get())); + config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName, + fs_.get())); - ASSERT_OK( - RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( + config_options, *parser.db_opt(), base_db_opt)); for (int c = 0; c < num_cf; ++c) { const auto* cf_opt = parser.GetCFOptions(cf_names[c]); ASSERT_NE(cf_opt, nullptr); ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( - base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c)))); + config_options, base_cf_opts[c], *cf_opt, + &(parser.cf_opt_maps()->at(c)))); } // Further verify pointer-typed options @@ -1651,7 +2607,8 @@ TEST_F(OptionsParserTest, DumpAndParse) { base_db_opt.max_open_files++; ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( - base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get())); + config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName, + fs_.get())); for (int c = 0; c < num_cf; ++c) { if (base_cf_opts[c].compaction_filter) { @@ -1675,8 +2632,8 @@ TEST_F(OptionsParserTest, DifferentDefault) { kOptionsFileName, fs_.get())); RocksDBOptionsParser parser; - ASSERT_OK( - parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */)); + ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, + 4096 /* readahead_size */)); { Options old_default_opts; @@ -1757,10 +2714,16 @@ class OptionsSanityCheckTest : public OptionsParserTest { protected: Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts, - OptionsSanityCheckLevel level) { + ConfigOptions::SanityLevel level, + bool input_strings_escaped = true) { + ConfigOptions config_options; + config_options.sanity_level = level; + config_options.ignore_unknown_options = false; + config_options.input_strings_escaped = input_strings_escaped; + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( - DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, fs_.get(), - level); + config_options, DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, + fs_.get()); } Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) { @@ -1782,7 +2745,8 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { // default ColumnFamilyOptions { ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); } // prefix_extractor @@ -1790,59 +2754,73 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { // Okay to change prefix_extractor form nullptr to non-nullptr ASSERT_EQ(opts.prefix_extractor.get(), nullptr); opts.prefix_extractor.reset(NewCappedPrefixTransform(10)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); // use same prefix extractor but with different parameter opts.prefix_extractor.reset(NewCappedPrefixTransform(15)); - // expect pass only in kSanityLevelLooselyCompatible - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + // expect pass only in + // ConfigOptions::kSanityLevelLooselyCompatible + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // repeat the test with FixedPrefixTransform opts.prefix_extractor.reset(NewFixedPrefixTransform(10)); - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // persist the change of prefix_extractor ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); // use same prefix extractor but with different parameter opts.prefix_extractor.reset(NewFixedPrefixTransform(15)); - // expect pass only in kSanityLevelLooselyCompatible - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + // expect pass only in + // ConfigOptions::kSanityLevelLooselyCompatible + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // Change prefix extractor from non-nullptr to nullptr opts.prefix_extractor.reset(); // expect pass as it's safe to change prefix_extractor // from non-null to null - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); } // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); // table_factory { for (int tb = 0; tb <= 2; ++tb) { // change the table factory opts.table_factory.reset(test::RandomTableFactory(&rnd, tb)); - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); } } @@ -1850,32 +2828,38 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { { // Test when going from nullptr -> merge operator opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); for (int test = 0; test < 5; ++test) { // change the merge operator opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); } // Test when going from merge operator -> nullptr opts.merge_operator = nullptr; - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); } // compaction_filter @@ -1883,12 +2867,15 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { for (int test = 0; test < 5; ++test) { // change the compaction filter opts.compaction_filter = test::RandomCompactionFilter(&rnd); - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); delete opts.compaction_filter; opts.compaction_filter = nullptr; } @@ -1900,12 +2887,15 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { // change the compaction filter factory opts.compaction_filter_factory.reset( test::RandomCompactionFilterFactory(&rnd)); - ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); // persist the change ASSERT_OK(PersistCFOptions(opts)); - ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); } } } diff --git a/src.mk b/src.mk index 825499146..a17e72949 100644 --- a/src.mk +++ b/src.mk @@ -496,6 +496,7 @@ JNI_NATIVE_SOURCES = \ java/rocksjni/comparator.cc \ java/rocksjni/comparatorjnicallback.cc \ java/rocksjni/compression_options.cc \ + java/rocksjni/config_options.cc \ java/rocksjni/env.cc \ java/rocksjni/env_options.cc \ java/rocksjni/ingest_external_file_options.cc \ diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index b79f8e7de..d2d507b3e 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -7,19 +7,22 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/block_based_table_factory.h" + #include -#include +#include #include #include #include "options/options_helper.h" +#include "options/options_parser.h" +#include "options/options_sanity_check.h" #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/convenience.h" #include "rocksdb/flush_block_policy.h" #include "table/block_based/block_based_table_builder.h" -#include "table/block_based/block_based_table_factory.h" #include "table/block_based/block_based_table_reader.h" #include "table/format.h" #include "util/mutexlock.h" @@ -536,17 +539,16 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const { } #ifndef ROCKSDB_LITE - Status BlockBasedTableFactory::GetOptionString( - std::string* opt_string, const std::string& delimiter) const { + const ConfigOptions& config_options, std::string* opt_string) const { assert(opt_string); opt_string->clear(); - return GetStringFromStruct(opt_string, &table_options_, - block_based_table_type_info, delimiter); + return GetStringFromStruct(config_options, &table_options_, + block_based_table_type_info, opt_string); } #else Status BlockBasedTableFactory::GetOptionString( - std::string* /*opt_string*/, const std::string& /*delimiter*/) const { + const ConfigOptions& /*opts*/, std::string* /*opt_string*/) const { return Status::OK(); } #endif // !ROCKSDB_LITE @@ -557,14 +559,14 @@ const BlockBasedTableOptions& BlockBasedTableFactory::table_options() const { #ifndef ROCKSDB_LITE namespace { -std::string ParseBlockBasedTableOption(const std::string& name, +std::string ParseBlockBasedTableOption(const ConfigOptions& config_options, + const std::string& name, const std::string& org_value, - BlockBasedTableOptions* new_options, - bool input_strings_escaped = false, - bool ignore_unknown_options = false) { - const std::string& value = - input_strings_escaped ? UnescapeOptionString(org_value) : org_value; - if (!input_strings_escaped) { + BlockBasedTableOptions* new_options) { + const std::string& value = config_options.input_strings_escaped + ? UnescapeOptionString(org_value) + : org_value; + if (!config_options.input_strings_escaped) { // if the input string is not escaped, it means this function is // invoked from SetOptions, which takes the old format. if (name == "block_cache" || name == "block_cache_compressed") { @@ -615,7 +617,7 @@ std::string ParseBlockBasedTableOption(const std::string& name, } const auto iter = block_based_table_type_info.find(name); if (iter == block_based_table_type_info.end()) { - if (ignore_unknown_options) { + if (config_options.ignore_unknown_options) { return ""; } else { return "Unrecognized option"; @@ -634,14 +636,23 @@ std::string ParseBlockBasedTableOption(const std::string& name, Status GetBlockBasedTableOptionsFromString( const BlockBasedTableOptions& table_options, const std::string& opts_str, BlockBasedTableOptions* new_table_options) { + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + return GetBlockBasedTableOptionsFromString(config_options, table_options, + opts_str, new_table_options); +} +Status GetBlockBasedTableOptionsFromString( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, const std::string& opts_str, + BlockBasedTableOptions* new_table_options) { std::unordered_map opts_map; Status s = StringToMap(opts_str, &opts_map); if (!s.ok()) { return s; } - - return GetBlockBasedTableOptionsFromMap(table_options, opts_map, - new_table_options); + return GetBlockBasedTableOptionsFromMap(config_options, table_options, + opts_map, new_table_options); } Status GetBlockBasedTableOptionsFromMap( @@ -649,18 +660,31 @@ Status GetBlockBasedTableOptionsFromMap( const std::unordered_map& opts_map, BlockBasedTableOptions* new_table_options, bool input_strings_escaped, bool ignore_unknown_options) { + ConfigOptions config_options; + config_options.input_strings_escaped = input_strings_escaped; + config_options.ignore_unknown_options = ignore_unknown_options; + + return GetBlockBasedTableOptionsFromMap(config_options, table_options, + opts_map, new_table_options); +} + +Status GetBlockBasedTableOptionsFromMap( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, + const std::unordered_map& opts_map, + BlockBasedTableOptions* new_table_options) { assert(new_table_options); *new_table_options = table_options; for (const auto& o : opts_map) { auto error_message = ParseBlockBasedTableOption( - o.first, o.second, new_table_options, input_strings_escaped, - ignore_unknown_options); + config_options, o.first, o.second, new_table_options); if (error_message != "") { const auto iter = block_based_table_type_info.find(o.first); if (iter == block_based_table_type_info.end() || - !input_strings_escaped || // !input_strings_escaped indicates - // the old API, where everything is - // parsable. + !config_options + .input_strings_escaped || // !input_strings_escaped indicates + // the old API, where everything is + // parsable. (!iter->second.IsByName() && !iter->second.IsDeprecated())) { // Restore "new_options" to the default "base_options". *new_table_options = table_options; @@ -672,12 +696,11 @@ Status GetBlockBasedTableOptionsFromMap( return Status::OK(); } -Status VerifyBlockBasedTableFactory( - const BlockBasedTableFactory* base_tf, - const BlockBasedTableFactory* file_tf, - OptionsSanityCheckLevel sanity_check_level) { +Status VerifyBlockBasedTableFactory(const ConfigOptions& config_options, + const BlockBasedTableFactory* base_tf, + const BlockBasedTableFactory* file_tf) { if ((base_tf != nullptr) != (file_tf != nullptr) && - sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone) { + config_options.sanity_level > ConfigOptions::kSanityLevelNone) { return Status::Corruption( "[RocksDBOptionsParser]: Inconsistent TableFactory class type"); } @@ -695,7 +718,7 @@ Status VerifyBlockBasedTableFactory( // contain random values since they might not be initialized continue; } - if (BBTOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (BBTOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) { if (!AreEqualOptions(reinterpret_cast(&base_opt), reinterpret_cast(&file_opt), pair.second, pair.first, nullptr)) { diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h index 6bc01cee6..38bf2395b 100644 --- a/table/block_based/block_based_table_factory.h +++ b/table/block_based/block_based_table_factory.h @@ -14,13 +14,11 @@ #include #include "db/dbformat.h" -#include "options/options_helper.h" -#include "options/options_parser.h" #include "rocksdb/flush_block_policy.h" #include "rocksdb/table.h" namespace ROCKSDB_NAMESPACE { - +struct ConfigOptions; struct EnvOptions; class BlockBasedTableBuilder; @@ -66,8 +64,8 @@ class BlockBasedTableFactory : public TableFactory { std::string GetPrintableTableOptions() const override; - Status GetOptionString(std::string* opt_string, - const std::string& delimiter) const override; + Status GetOptionString(const ConfigOptions& config_options, + std::string* opt_string) const override; const BlockBasedTableOptions& table_options() const; @@ -89,9 +87,7 @@ extern const std::string kPropFalse; #ifndef ROCKSDB_LITE extern Status VerifyBlockBasedTableFactory( - const BlockBasedTableFactory* base_tf, - const BlockBasedTableFactory* file_tf, - OptionsSanityCheckLevel sanity_check_level); - + const ConfigOptions& config_options, const BlockBasedTableFactory* base_tf, + const BlockBasedTableFactory* file_tf); #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc index 9a925de83..e7f40d079 100644 --- a/table/block_based/block_based_table_reader.cc +++ b/table/block_based/block_based_table_reader.cc @@ -7,6 +7,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "table/block_based/block_based_table_reader.h" + #include #include #include @@ -16,10 +17,10 @@ #include "db/dbformat.h" #include "db/pinned_iterators_manager.h" - #include "file/file_prefetch_buffer.h" #include "file/random_access_file_reader.h" - +#include "monitoring/perf_context_imp.h" +#include "options/options_helper.h" #include "rocksdb/cache.h" #include "rocksdb/comparator.h" #include "rocksdb/env.h" @@ -30,7 +31,6 @@ #include "rocksdb/statistics.h" #include "rocksdb/table.h" #include "rocksdb/table_properties.h" - #include "table/block_based/binary_search_index_reader.h" #include "table/block_based/block.h" #include "table/block_based/block_based_filter_block.h" diff --git a/table/cuckoo/cuckoo_table_factory.h b/table/cuckoo/cuckoo_table_factory.h index 1b5b6c200..6a8c36981 100644 --- a/table/cuckoo/cuckoo_table_factory.h +++ b/table/cuckoo/cuckoo_table_factory.h @@ -79,8 +79,8 @@ class CuckooTableFactory : public TableFactory { void* GetOptions() override { return &table_options_; } - Status GetOptionString(std::string* /*opt_string*/, - const std::string& /*delimiter*/) const override { + Status GetOptionString(const ConfigOptions& /*config_options*/, + std::string* /*opt_string*/) const override { return Status::OK(); } diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index df296cdff..4759eb975 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -7,7 +7,9 @@ #include "table/plain/plain_table_factory.h" #include + #include + #include "db/dbformat.h" #include "options/options_helper.h" #include "port/port.h" @@ -117,12 +119,24 @@ const PlainTableOptions& PlainTableFactory::table_options() const { Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options, const std::string& opts_str, PlainTableOptions* new_table_options) { + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + return GetPlainTableOptionsFromString(config_options, table_options, opts_str, + new_table_options); +} + +Status GetPlainTableOptionsFromString(const ConfigOptions& config_options, + const PlainTableOptions& table_options, + const std::string& opts_str, + PlainTableOptions* new_table_options) { std::unordered_map opts_map; Status s = StringToMap(opts_str, &opts_map); if (!s.ok()) { return s; } - return GetPlainTableOptionsFromMap(table_options, opts_map, + + return GetPlainTableOptionsFromMap(config_options, table_options, opts_map, new_table_options); } @@ -190,16 +204,16 @@ Status GetMemTableRepFactoryFromString( return Status::OK(); } -std::string ParsePlainTableOptions(const std::string& name, +std::string ParsePlainTableOptions(const ConfigOptions& config_options, + const std::string& name, const std::string& org_value, - PlainTableOptions* new_options, - bool input_strings_escaped = false, - bool ignore_unknown_options = false) { - const std::string& value = - input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + PlainTableOptions* new_options) { + const std::string& value = config_options.input_strings_escaped + ? UnescapeOptionString(org_value) + : org_value; const auto iter = plain_table_type_info.find(name); if (iter == plain_table_type_info.end()) { - if (ignore_unknown_options) { + if (config_options.ignore_unknown_options) { return ""; } else { return "Unrecognized option"; @@ -218,18 +232,30 @@ Status GetPlainTableOptionsFromMap( const PlainTableOptions& table_options, const std::unordered_map& opts_map, PlainTableOptions* new_table_options, bool input_strings_escaped, - bool /*ignore_unknown_options*/) { + bool ignore_unknown_options) { + ConfigOptions config_options; + config_options.input_strings_escaped = input_strings_escaped; + config_options.ignore_unknown_options = ignore_unknown_options; + return GetPlainTableOptionsFromMap(config_options, table_options, opts_map, + new_table_options); +} + +Status GetPlainTableOptionsFromMap( + const ConfigOptions& config_options, const PlainTableOptions& table_options, + const std::unordered_map& opts_map, + PlainTableOptions* new_table_options) { assert(new_table_options); *new_table_options = table_options; for (const auto& o : opts_map) { - auto error_message = ParsePlainTableOptions( - o.first, o.second, new_table_options, input_strings_escaped); + auto error_message = ParsePlainTableOptions(config_options, o.first, + o.second, new_table_options); if (error_message != "") { const auto iter = plain_table_type_info.find(o.first); if (iter == plain_table_type_info.end() || - !input_strings_escaped || // !input_strings_escaped indicates - // the old API, where everything is - // parsable. + !config_options + .input_strings_escaped || // !input_strings_escaped indicates + // the old API, where everything is + // parsable. (!iter->second.IsByName() && !iter->second.IsDeprecated())) { // Restore "new_options" to the default "base_options". *new_table_options = table_options; diff --git a/table/plain/plain_table_factory.h b/table/plain/plain_table_factory.h index cbd892a9e..282a24a5b 100644 --- a/table/plain/plain_table_factory.h +++ b/table/plain/plain_table_factory.h @@ -184,8 +184,8 @@ class PlainTableFactory : public TableFactory { void* GetOptions() override { return &table_options_; } - Status GetOptionString(std::string* /*opt_string*/, - const std::string& /*delimiter*/) const override { + Status GetOptionString(const ConfigOptions& /*config_options*/, + std::string* /*opt_string*/) const override { return Status::OK(); } diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index bfcd118bf..56d52a340 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -330,7 +330,6 @@ LDBCommand::LDBCommand(const std::map& options, is_db_ttl_(false), timestamp_(false), try_load_options_(false), - ignore_unknown_options_(false), create_if_missing_(false), option_map_(options), flags_(flags), @@ -363,13 +362,15 @@ LDBCommand::LDBCommand(const std::map& options, is_db_ttl_ = IsFlagPresent(flags, ARG_TTL); timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP); try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS); - ignore_unknown_options_ = IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS); + config_options_.ignore_unknown_options = + IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS); } void LDBCommand::OpenDB() { if (!create_if_missing_ && try_load_options_) { - Status s = LoadLatestOptions(db_path_, options_.env, &options_, - &column_families_, ignore_unknown_options_); + config_options_.env = options_.env; + Status s = LoadLatestOptions(config_options_, db_path_, &options_, + &column_families_); if (!s.ok() && !s.IsNotFound()) { // Option file exists but load option file error. std::string msg = s.ToString(); diff --git a/utilities/options/options_util.cc b/utilities/options/options_util.cc index 0719798e3..065211a84 100644 --- a/utilities/options/options_util.cc +++ b/utilities/options/options_util.cc @@ -10,7 +10,9 @@ #include "env/composite_env_wrapper.h" #include "file/filename.h" #include "options/options_parser.h" +#include "rocksdb/convenience.h" #include "rocksdb/options.h" +#include "table/block_based/block_based_table_factory.h" namespace ROCKSDB_NAMESPACE { Status LoadOptionsFromFile(const std::string& file_name, Env* env, @@ -18,10 +20,22 @@ Status LoadOptionsFromFile(const std::string& file_name, Env* env, std::vector* cf_descs, bool ignore_unknown_options, std::shared_ptr* cache) { + ConfigOptions config_options; + config_options.ignore_unknown_options = ignore_unknown_options; + config_options.input_strings_escaped = true; + config_options.env = env; + + return LoadOptionsFromFile(config_options, file_name, db_options, cf_descs, + cache); +} + +Status LoadOptionsFromFile(const ConfigOptions& config_options, + const std::string& file_name, DBOptions* db_options, + std::vector* cf_descs, + std::shared_ptr* cache) { RocksDBOptionsParser parser; - LegacyFileSystemWrapper fs(env); - Status s = parser.Parse(file_name, &fs, ignore_unknown_options, - 0 /* file_readahead_size */); + LegacyFileSystemWrapper fs(config_options.env); + Status s = parser.Parse(config_options, file_name, &fs); if (!s.ok()) { return s; } @@ -76,21 +90,48 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env, std::vector* cf_descs, bool ignore_unknown_options, std::shared_ptr* cache) { + ConfigOptions config_options; + config_options.ignore_unknown_options = ignore_unknown_options; + config_options.input_strings_escaped = true; + config_options.env = env; + + return LoadLatestOptions(config_options, dbpath, db_options, cf_descs, cache); +} + +Status LoadLatestOptions(const ConfigOptions& config_options, + const std::string& dbpath, DBOptions* db_options, + std::vector* cf_descs, + std::shared_ptr* cache) { std::string options_file_name; - Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); + Status s = + GetLatestOptionsFileName(dbpath, config_options.env, &options_file_name); if (!s.ok()) { return s; } - return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, db_options, - cf_descs, ignore_unknown_options, cache); + return LoadOptionsFromFile(config_options, dbpath + "/" + options_file_name, + db_options, cf_descs, cache); } Status CheckOptionsCompatibility( const std::string& dbpath, Env* env, const DBOptions& db_options, const std::vector& cf_descs, bool ignore_unknown_options) { + ConfigOptions config_options; + config_options.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + config_options.ignore_unknown_options = ignore_unknown_options; + config_options.input_strings_escaped = true; + config_options.env = env; + return CheckOptionsCompatibility(config_options, dbpath, db_options, + cf_descs); +} + +Status CheckOptionsCompatibility( + const ConfigOptions& config_options, const std::string& dbpath, + const DBOptions& db_options, + const std::vector& cf_descs) { std::string options_file_name; - Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); + Status s = + GetLatestOptionsFileName(dbpath, config_options.env, &options_file_name); if (!s.ok()) { return s; } @@ -102,12 +143,12 @@ Status CheckOptionsCompatibility( cf_opts.push_back(cf_desc.options); } - const OptionsSanityCheckLevel kDefaultLevel = kSanityLevelLooselyCompatible; - LegacyFileSystemWrapper fs(env); + LegacyFileSystemWrapper fs(config_options.env); return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( - db_options, cf_names, cf_opts, dbpath + "/" + options_file_name, &fs, - kDefaultLevel, ignore_unknown_options); + + config_options, db_options, cf_names, cf_opts, + dbpath + "/" + options_file_name, &fs); } } // namespace ROCKSDB_NAMESPACE diff --git a/utilities/options/options_util_test.cc b/utilities/options/options_util_test.cc index 30ad76a99..0062ad1cd 100644 --- a/utilities/options/options_util_test.cc +++ b/utilities/options/options_util_test.cc @@ -5,15 +5,16 @@ #ifndef ROCKSDB_LITE -#include +#include "rocksdb/utilities/options_util.h" #include +#include #include #include "options/options_parser.h" +#include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/table.h" -#include "rocksdb/utilities/options_util.h" #include "test_util/testharness.h" #include "test_util/testutil.h" #include "util/random.h" @@ -67,23 +68,26 @@ TEST_F(OptionsUtilTest, SaveAndLoad) { std::vector loaded_cf_descs; ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, &loaded_cf_descs)); - - ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); + ConfigOptions exact; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, db_opt, loaded_db_opt)); test::RandomInitDBOptions(&db_opt, &rnd_); - ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); + ASSERT_NOK( + RocksDBOptionsParser::VerifyDBOptions(exact, db_opt, loaded_db_opt)); for (size_t i = 0; i < kCFCount; ++i) { ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name); ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( - cf_opts[i], loaded_cf_descs[i].options)); + exact, cf_opts[i], loaded_cf_descs[i].options)); if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory( - cf_opts[i].table_factory.get(), + exact, cf_opts[i].table_factory.get(), loaded_cf_descs[i].options.table_factory.get())); } test::RandomInitCFOptions(&cf_opts[i], db_opt, &rnd_); ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( - cf_opts[i], loaded_cf_descs[i].options)); + exact, cf_opts[i], loaded_cf_descs[i].options)); } for (size_t i = 0; i < kCFCount; ++i) { @@ -125,6 +129,25 @@ TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) { PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, fs_.get()); DBOptions loaded_db_opt; std::vector loaded_cf_descs; + + ConfigOptions config_options; + config_options.ignore_unknown_options = false; + config_options.input_strings_escaped = true; + config_options.env = env_.get(); + ASSERT_OK(LoadOptionsFromFile(config_options, kFileName, &loaded_db_opt, + &loaded_cf_descs, &cache)); + for (size_t i = 0; i < loaded_cf_descs.size(); i++) { + if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { + auto* loaded_bbt_opt = reinterpret_cast( + loaded_cf_descs[i].options.table_factory->GetOptions()); + // Expect the same cache will be loaded + if (loaded_bbt_opt != nullptr) { + ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); + } + } + } + + // Test the old interface ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, &loaded_cf_descs, false, &cache)); for (size_t i = 0; i < loaded_cf_descs.size(); i++) { @@ -170,8 +193,8 @@ class DummyTableFactory : public TableFactory { std::string GetPrintableTableOptions() const override { return ""; } - Status GetOptionString(std::string* /*opt_string*/, - const std::string& /*delimiter*/) const override { + Status GetOptionString(const ConfigOptions& /*opts*/, + std::string* /*opt_string*/) const override { return Status::OK(); } }; @@ -248,9 +271,13 @@ TEST_F(OptionsUtilTest, SanityCheck) { } delete db; + ConfigOptions config_options; + config_options.ignore_unknown_options = false; + config_options.input_strings_escaped = true; + config_options.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; // perform sanity check ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); ASSERT_GE(kCFCount, 5); // merge operator @@ -261,15 +288,15 @@ TEST_F(OptionsUtilTest, SanityCheck) { ASSERT_NE(merge_op.get(), nullptr); cf_descs[0].options.merge_operator.reset(); ASSERT_NOK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); cf_descs[0].options.merge_operator.reset(new DummyMergeOperator()); ASSERT_NOK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); cf_descs[0].options.merge_operator = merge_op; ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); } // prefix extractor @@ -281,15 +308,15 @@ TEST_F(OptionsUtilTest, SanityCheck) { ASSERT_NE(prefix_extractor, nullptr); cf_descs[1].options.prefix_extractor.reset(); ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); cf_descs[1].options.prefix_extractor.reset(new DummySliceTransform()); ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); cf_descs[1].options.prefix_extractor = prefix_extractor; ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); } // prefix extractor nullptr case @@ -301,16 +328,16 @@ TEST_F(OptionsUtilTest, SanityCheck) { ASSERT_EQ(prefix_extractor, nullptr); cf_descs[0].options.prefix_extractor.reset(); ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); // It's okay to change prefix_extractor from nullptr to non-nullptr cf_descs[0].options.prefix_extractor.reset(new DummySliceTransform()); ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); cf_descs[0].options.prefix_extractor = prefix_extractor; ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); } // comparator @@ -320,11 +347,11 @@ TEST_F(OptionsUtilTest, SanityCheck) { auto* prev_comparator = cf_descs[2].options.comparator; cf_descs[2].options.comparator = &comparator; ASSERT_NOK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); cf_descs[2].options.comparator = prev_comparator; ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); } // table factory @@ -335,11 +362,11 @@ TEST_F(OptionsUtilTest, SanityCheck) { ASSERT_NE(table_factory, nullptr); cf_descs[3].options.table_factory.reset(new DummyTableFactory()); ASSERT_NOK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); cf_descs[3].options.table_factory = table_factory; ASSERT_OK( - CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs)); } }