From e11f676e344eb66fae0df2bf9a301b70a222e9dd Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Thu, 12 Nov 2015 06:52:43 -0800 Subject: [PATCH] Add OptionsUtil::LoadOptionsFromFile() API Summary: This patch adds OptionsUtil::LoadOptionsFromFile() and OptionsUtil::LoadLatestOptionsFromDB(), which allow developers to construct DBOptions and ColumnFamilyOptions from a RocksDB options file. Note that most pointer-typed options such as merge_operator will not be constructed. With this API, developers no longer need to remember all the options in order to reopen an existing rocksdb instance like the following: DBOptions db_options; std::vector cf_names; std::vector cf_opts; // Load primitive-typed options from an existing DB OptionsUtil::LoadLatestOptionsFromDB( dbname, &db_options, &cf_names, &cf_opts); // Initialize necessary pointer-typed options cf_opts[0].merge_operator.reset(new MyMergeOperator()); ... // Construct the vector of ColumnFamilyDescriptor std::vector cf_descs; for (size_t i = 0; i < cf_opts.size(); ++i) { cf_descs.emplace_back(cf_names[i], cf_opts[i]); } // Open the DB DB* db = nullptr; std::vector cf_handles; auto s = DB::Open(db_options, dbname, cf_descs, &handles, &db); Test Plan: Augment existing tests in column_family_test options_test db_test Reviewers: igor, IslamAbdelRahman, sdong, anthony Reviewed By: anthony Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D49095 --- CMakeLists.txt | 2 + HISTORY.md | 1 + Makefile | 4 + examples/Makefile | 3 + examples/options_file_example.cc | 113 ++++++ include/rocksdb/table.h | 16 + include/rocksdb/utilities/options_util.h | 65 ++++ src.mk | 2 + table/block_based_table_factory.h | 2 + table/cuckoo_table_factory.h | 4 +- table/plain_table_factory.h | 2 + util/options_helper.cc | 167 ++++++--- util/options_helper.h | 20 + util/options_parser.h | 6 +- util/options_test.cc | 451 +---------------------- util/testutil.cc | 208 ++++++++++- util/testutil.h | 259 +++++++++++++ utilities/options/options_util.cc | 76 ++++ utilities/options/options_util_test.cc | 102 +++++ 19 files changed, 1020 insertions(+), 483 deletions(-) create mode 100644 examples/options_file_example.cc create mode 100644 include/rocksdb/utilities/options_util.h create mode 100644 utilities/options/options_util.cc create mode 100644 utilities/options/options_util_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index ad99eba36..ed9fde0b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -250,6 +250,7 @@ set(SOURCES utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/put.cc utilities/merge_operators/uint64add.cc + utilities/options/options_util.cc utilities/redis/redis_lists.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc @@ -382,6 +383,7 @@ set(TESTS utilities/geodb/geodb_test.cc utilities/memory/memory_test.cc utilities/merge_operators/string_append/stringappend_test.cc + utilities/options_util_test.cc utilities/redis/redis_lists_test.cc utilities/spatialdb/spatial_db_test.cc utilities/table_properties_collectors/compact_on_deletion_collector_test.cc diff --git a/HISTORY.md b/HISTORY.md index a54c13da4..a6a4018db 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,7 @@ * Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families. * Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances. * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions. +* Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance. ### Public API Changes * CompactionFilter::Context includes information of Column Family ID * The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. diff --git a/Makefile b/Makefile index aae039663..02ffd5653 100644 --- a/Makefile +++ b/Makefile @@ -303,6 +303,7 @@ TESTS = \ rate_limiter_test \ delete_scheduler_test \ options_test \ + options_util_test \ event_logger_test \ cuckoo_table_builder_test \ cuckoo_table_reader_test \ @@ -929,6 +930,9 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS) options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(AM_LINK) + event_logger_test: util/event_logger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) diff --git a/examples/Makefile b/examples/Makefile index fe82d11cd..5bf0577ff 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -28,6 +28,9 @@ optimistic_transaction_example: librocksdb optimistic_transaction_example.cc transaction_example: librocksdb transaction_example.cc $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) +options_file_example: librocksdb options_file_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + clean: rm -rf ./simple_example ./column_families_example ./compact_files_example ./compaction_filter_example ./c_simple_example c_simple_example.o ./optimistic_transaction_example ./transaction_example diff --git a/examples/options_file_example.cc b/examples/options_file_example.cc new file mode 100644 index 000000000..916ff02f3 --- /dev/null +++ b/examples/options_file_example.cc @@ -0,0 +1,113 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// This file demonstrates how to use the utility functions defined in +// rocksdb/utilities/options_util.h to open a rocksdb database without +// remembering all the rocksdb options. +#include +#include +#include + +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/options_util.h" + +using namespace rocksdb; + +std::string kDBPath = "/tmp/rocksdb_options_file_example"; + +namespace { +// A dummy compaction filter +class DummyCompactionFilter : public CompactionFilter { + public: + virtual ~DummyCompactionFilter() {} + virtual bool Filter(int level, const Slice& key, const Slice& existing_value, + std::string* new_value, bool* value_changed) const { + return false; + } + virtual const char* Name() const { return "DummyCompactionFilter"; } +}; + +} // namespace + +int main() { + DBOptions db_opt; + db_opt.create_if_missing = true; + + std::vector cf_descs; + cf_descs.push_back({kDefaultColumnFamilyName, ColumnFamilyOptions()}); + cf_descs.push_back({"new_cf", ColumnFamilyOptions()}); + + // initialize BlockBasedTableOptions + auto cache = NewLRUCache(1 * 1024 * 1024 * 1024); + BlockBasedTableOptions bbt_opts; + bbt_opts.block_size = 32 * 1024; + bbt_opts.block_cache = cache; + + // initialize column families options + std::unique_ptr compaction_filter; + compaction_filter.reset(new DummyCompactionFilter()); + cf_descs[0].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); + cf_descs[0].options.compaction_filter = compaction_filter.get(); + cf_descs[1].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); + + // destroy and open DB + DB* db; + Status s = DestroyDB(kDBPath, Options(db_opt, cf_descs[0].options)); + assert(s.ok()); + s = DB::Open(Options(db_opt, cf_descs[0].options), kDBPath, &db); + assert(s.ok()); + + // Create column family, and rocksdb will persist the options. + ColumnFamilyHandle* cf; + s = db->CreateColumnFamily(ColumnFamilyOptions(), "new_cf", &cf); + assert(s.ok()); + + // close DB + delete cf; + delete db; + + // In the following code, we will reopen the rocksdb instance using + // the options file stored in the db directory. + + // Load the options file. + DBOptions loaded_db_opt; + std::vector loaded_cf_descs; + s = LoadLatestOptions(kDBPath, Env::Default(), &loaded_db_opt, + &loaded_cf_descs); + assert(s.ok()); + assert(loaded_db_opt.create_if_missing == db_opt.create_if_missing); + + // Initialize pointer options for each column family + for (size_t i = 0; i < loaded_cf_descs.size(); ++i) { + auto* loaded_bbt_opt = reinterpret_cast( + loaded_cf_descs[0].options.table_factory->GetOptions()); + // Expect the same as BlockBasedTableOptions will be loaded form file. + assert(loaded_bbt_opt->block_size == bbt_opts.block_size); + // However, block_cache needs to be manually initialized as documented + // in rocksdb/utilities/options_util.h. + loaded_bbt_opt->block_cache = cache; + } + // In addition, as pointer options are initialized with default value, + // we need to properly initialized all the pointer options if non-defalut + // values are used before calling DB::Open(). + assert(loaded_cf_descs[0].options.compaction_filter == nullptr); + loaded_cf_descs[0].options.compaction_filter = compaction_filter.get(); + + // reopen the db using the loaded options. + std::vector handles; + s = DB::Open(loaded_db_opt, kDBPath, loaded_cf_descs, &handles, &db); + assert(s.ok()); + + // close DB + for (auto* handle : handles) { + delete handle; + } + delete db; +} diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 932c77df5..060c400e6 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -402,6 +402,22 @@ class TableFactory { // Return a string that contains printable format of table configurations. // RocksDB prints configurations at DB Open(). virtual std::string GetPrintableTableOptions() const = 0; + + // Returns the raw pointer of the table options that is used by this + // TableFactory, or nullptr if this function is not supported. + // Since the return value is a raw pointer, the TableFactory owns the + // pointer and the caller should not delete the pointer. + // + // In certan case, it is desirable to alter the underlying options when the + // TableFactory is not used by any open DB by casting the returned pointer + // to the right class. For instance, if BlockBasedTableFactory is used, + // then the pointer can be casted to BlockBasedTableOptions. + // + // Note that changing the underlying TableFactory options while the + // TableFactory is currently used by any open DB is undefined behavior. + // Developers should use DB::SetOption() instead to dynamically change + // options while the DB is open. + virtual void* GetOptions() { return nullptr; } }; #ifndef ROCKSDB_LITE diff --git a/include/rocksdb/utilities/options_util.h b/include/rocksdb/utilities/options_util.h new file mode 100644 index 000000000..e1b4c3fda --- /dev/null +++ b/include/rocksdb/utilities/options_util.h @@ -0,0 +1,65 @@ +// Copyright (c) 2015, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// This file contains utility functions for RocksDB Options. +#pragma once + +#ifndef ROCKSDB_LITE + +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" + +namespace rocksdb { +// Constructs the DBOptions and ColumnFamilyDescriptors by loading the +// latest RocksDB options file stored in the specified rocksdb database. +// +// Note that the all the pointer options (except table_factory, which will +// be described in more details below) will be initialized with the default +// values. Developers can further initialize them after this function call. +// Below is an example list of pointer options which will be initialized +// +// * env +// * memtable_factory +// * compaction_filter_factory +// * prefix_extractor +// * comparator +// * merge_operator +// * compaction_filter +// +// For table_factory, this function further supports deserializing +// BlockBasedTableFactory and its BlockBasedTableOptions except the +// pointer options of BlockBasedTableOptions (flush_block_policy_factory, +// block_cache, and block_cache_compressed), which will be initialized with +// default values. Developers can further specify these three options by +// casting the return value of TableFactoroy::GetOptions() to +// BlockBasedTableOptions and making necessary changes. +// +// examples/options_file_example.cc demonstrates how to use this function +// to open a RocksDB instance. +// +// @see LoadOptionsFromFile +Status LoadLatestOptions(const std::string& dbpath, Env* env, + DBOptions* db_options, + std::vector* cf_descs); + +// Similar to LoadLatestOptions, this function constructs the DBOptions +// and ColumnFamilyDescriptors based on the specified RocksDB Options file. +// +// @see LoadLatestOptions +Status LoadOptionsFromFile(const std::string& options_file_name, Env* env, + DBOptions* db_options, + std::vector* cf_descs); + +// Returns the latest options file name under the specified db path. +Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, + std::string* options_file_name); + +} // namespace rocksdb +#endif // !ROCKSDB_LITE diff --git a/src.mk b/src.mk index 02fed6ed7..b6cc81cac 100644 --- a/src.mk +++ b/src.mk @@ -118,6 +118,7 @@ LIB_SOURCES = \ utilities/merge_operators/string_append/stringappend2.cc \ utilities/merge_operators/string_append/stringappend.cc \ utilities/merge_operators/uint64add.cc \ + utilities/options/options_util.cc \ utilities/redis/redis_lists.cc \ utilities/spatialdb/spatial_db.cc \ utilities/table_properties_collectors/compact_on_deletion_collector.cc \ @@ -254,6 +255,7 @@ TEST_BENCH_SOURCES = \ utilities/geodb/geodb_test.cc \ utilities/memory/memory_test.cc \ utilities/merge_operators/string_append/stringappend_test.cc \ + utilities/options_util_test.cc \ utilities/redis/redis_lists_test.cc \ utilities/spatialdb/spatial_db_test.cc \ utilities/table_properties_collectors/compact_on_deletion_collector_test.cc \ diff --git a/table/block_based_table_factory.h b/table/block_based_table_factory.h index 522e31c22..98e622673 100644 --- a/table/block_based_table_factory.h +++ b/table/block_based_table_factory.h @@ -58,6 +58,8 @@ class BlockBasedTableFactory : public TableFactory { const BlockBasedTableOptions& GetTableOptions() const; + void* GetOptions() override { return &table_options_; } + private: BlockBasedTableOptions table_options_; }; diff --git a/table/cuckoo_table_factory.h b/table/cuckoo_table_factory.h index b28342c14..3f89ca86d 100644 --- a/table/cuckoo_table_factory.h +++ b/table/cuckoo_table_factory.h @@ -72,8 +72,10 @@ class CuckooTableFactory : public TableFactory { std::string GetPrintableTableOptions() const override; + void* GetOptions() override { return &table_options_; } + private: - const CuckooTableOptions table_options_; + CuckooTableOptions table_options_; }; } // namespace rocksdb diff --git a/table/plain_table_factory.h b/table/plain_table_factory.h index 0f5bf7937..301b07a06 100644 --- a/table/plain_table_factory.h +++ b/table/plain_table_factory.h @@ -168,6 +168,8 @@ class PlainTableFactory : public TableFactory { return Status::OK(); } + void* GetOptions() override { return &table_options_; } + private: PlainTableOptions table_options_; }; diff --git a/util/options_helper.cc b/util/options_helper.cc index 3ff13ae31..3c545e115 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -548,7 +548,16 @@ bool SerializeSingleOptionHelper(const char* opt_address, case OptionType::kComparator: { // it's a const pointer of const Comparator* const auto* ptr = reinterpret_cast(opt_address); - *value = *ptr ? (*ptr)->Name() : kNullptrString; + // Since the user-specified comparator will be wrapped by + // InternalKeyComparator, we should persist the user-specified one + // instead of InternalKeyComparator. + const auto* internal_comparator = + dynamic_cast(*ptr); + if (internal_comparator != nullptr) { + *value = internal_comparator->user_comparator()->Name(); + } else { + *value = *ptr ? (*ptr)->Name() : kNullptrString; + } break; } case OptionType::kCompactionFilter: { @@ -804,10 +813,10 @@ Status StringToMap(const std::string& opts_str, return Status::OK(); } -bool ParseColumnFamilyOption(const std::string& name, - const std::string& org_value, - ColumnFamilyOptions* new_options, - bool input_strings_escaped = false) { +Status ParseColumnFamilyOption(const std::string& name, + const std::string& org_value, + ColumnFamilyOptions* new_options, + bool input_strings_escaped = false) { const std::string& value = input_strings_escaped ? UnescapeOptionString(org_value) : org_value; try { @@ -837,7 +846,8 @@ bool ParseColumnFamilyOption(const std::string& name, Status table_opt_s = GetBlockBasedTableOptionsFromString( base_table_options, value, &table_opt); if (!table_opt_s.ok()) { - return false; + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); } new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); } else if (name == "plain_table_factory") { @@ -851,50 +861,66 @@ bool ParseColumnFamilyOption(const std::string& name, Status table_opt_s = GetPlainTableOptionsFromString( base_table_options, value, &table_opt); if (!table_opt_s.ok()) { - return false; + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); } new_options->table_factory.reset(NewPlainTableFactory(table_opt)); } else if (name == "compression_opts") { size_t start = 0; size_t end = value.find(':'); if (end == std::string::npos) { - return false; + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); } new_options->compression_opts.window_bits = ParseInt(value.substr(start, end - start)); start = end + 1; end = value.find(':', start); if (end == std::string::npos) { - return false; + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); } new_options->compression_opts.level = ParseInt(value.substr(start, end - start)); start = end + 1; if (start >= value.size()) { - return false; + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); } new_options->compression_opts.strategy = ParseInt(value.substr(start, value.size() - start)); - } else if (name == "compaction_options_universal") { - // TODO(ljin): add support - return false; } else if (name == "compaction_options_fifo") { new_options->compaction_options_fifo.max_table_files_size = ParseUint64(value); } else { auto iter = cf_options_type_info.find(name); if (iter == cf_options_type_info.end()) { - return false; + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); } const auto& opt_info = iter->second; - return ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.offset, opt_info.type, - value); + if (ParseOptionHelper( + reinterpret_cast(new_options) + opt_info.offset, + opt_info.type, value)) { + return Status::OK(); + } + switch (opt_info.verification) { + case OptionVerificationType::kByName: + return Status::NotSupported( + "Deserializing the specified CF option " + name + + " is not supported"); + case OptionVerificationType::kDeprecated: + return Status::OK(); + default: + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); + } } } catch (const std::exception&) { - return false; + return Status::InvalidArgument( + "unable to parse the specified option " + name); } - return true; + return Status::OK(); } bool SerializeSingleDBOption(std::string* opt_string, @@ -1037,8 +1063,10 @@ Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf, return Status::OK(); } -bool ParseDBOption(const std::string& name, const std::string& org_value, - DBOptions* new_options, bool input_strings_escaped = false) { +Status ParseDBOption(const std::string& name, + const std::string& org_value, + DBOptions* new_options, + bool input_strings_escaped = false) { const std::string& value = input_strings_escaped ? UnescapeOptionString(org_value) : org_value; try { @@ -1048,20 +1076,30 @@ bool ParseDBOption(const std::string& name, const std::string& org_value, } else { auto iter = db_options_type_info.find(name); if (iter == db_options_type_info.end()) { - return false; + return Status::InvalidArgument("Unrecognized option DBOptions:", name); } const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kByName && - opt_info.verification != OptionVerificationType::kDeprecated) { - return ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.offset, - opt_info.type, value); + if (ParseOptionHelper( + reinterpret_cast(new_options) + opt_info.offset, + opt_info.type, value)) { + return Status::OK(); + } + switch (opt_info.verification) { + case OptionVerificationType::kByName: + return Status::NotSupported( + "Deserializing the specified DB option " + name + + " is not supported"); + case OptionVerificationType::kDeprecated: + return Status::OK(); + default: + return Status::InvalidArgument( + "Unable to parse the specified DB option " + name); } } } catch (const std::exception&) { - return false; + return Status::InvalidArgument("Unable to parse DBOptions:", name); } - return true; + return Status::OK(); } std::string ParseBlockBasedTableOption(const std::string& name, @@ -1209,16 +1247,36 @@ Status GetColumnFamilyOptionsFromMap( const ColumnFamilyOptions& base_options, const std::unordered_map& opts_map, ColumnFamilyOptions* new_options, bool input_strings_escaped) { + return GetColumnFamilyOptionsFromMapInternal( + base_options, opts_map, new_options, input_strings_escaped); +} + +Status GetColumnFamilyOptionsFromMapInternal( + const ColumnFamilyOptions& base_options, + const std::unordered_map& opts_map, + ColumnFamilyOptions* new_options, bool input_strings_escaped, + std::vector* unsupported_options_names) { assert(new_options); *new_options = base_options; + if (unsupported_options_names) { + unsupported_options_names->clear(); + } for (const auto& o : opts_map) { - if (!ParseColumnFamilyOption(o.first, o.second, new_options, - input_strings_escaped)) { - auto iter = cf_options_type_info.find(o.first); - if (iter == cf_options_type_info.end() || - (iter->second.verification != OptionVerificationType::kByName && - iter->second.verification != OptionVerificationType::kDeprecated)) { - return Status::InvalidArgument("Can't parse option " + o.first); + auto s = ParseColumnFamilyOption(o.first, o.second, new_options, + input_strings_escaped); + if (!s.ok()) { + if (s.IsNotSupported()) { + // If the deserialization of the specified option is not supported + // and an output vector of unsupported_options is provided, then + // we log the name of the unsupported option and proceed. + if (unsupported_options_names != nullptr) { + unsupported_options_names->push_back(o.first); + } + // Note that we still return Status::OK in such case to maintain + // the backward compatibility in the old public API defined in + // rocksdb/convenience.h + } else { + return s; } } } @@ -1241,13 +1299,37 @@ Status GetDBOptionsFromMap( const DBOptions& base_options, const std::unordered_map& opts_map, DBOptions* new_options, bool input_strings_escaped) { + return GetDBOptionsFromMapInternal( + base_options, opts_map, new_options, input_strings_escaped); +} + +Status GetDBOptionsFromMapInternal( + const DBOptions& base_options, + const std::unordered_map& opts_map, + DBOptions* new_options, bool input_strings_escaped, + std::vector* unsupported_options_names) { assert(new_options); *new_options = base_options; + if (unsupported_options_names) { + unsupported_options_names->clear(); + } for (const auto& o : opts_map) { - if (!ParseDBOption(o.first, o.second, new_options, input_strings_escaped)) { - // Note that options with kDeprecated validation will pass ParseDBOption - // and will not hit the below statement. - return Status::InvalidArgument("Can't parse option " + o.first); + auto s = ParseDBOption(o.first, o.second, + new_options, input_strings_escaped); + if (!s.ok()) { + if (s.IsNotSupported()) { + // If the deserialization of the specified option is not supported + // and an output vector of unsupported_options is provided, then + // we log the name of the unsupported option and proceed. + if (unsupported_options_names != nullptr) { + unsupported_options_names->push_back(o.first); + } + // Note that we still return Status::OK in such case to maintain + // the backward compatibility in the old public API defined in + // rocksdb/convenience.h + } else { + return s; + } } } return Status::OK(); @@ -1275,8 +1357,9 @@ Status GetOptionsFromString(const Options& base_options, DBOptions new_db_options(base_options); ColumnFamilyOptions new_cf_options(base_options); for (const auto& o : opts_map) { - if (ParseDBOption(o.first, o.second, &new_db_options)) { - } else if (ParseColumnFamilyOption(o.first, o.second, &new_cf_options)) { + if (ParseDBOption(o.first, o.second, &new_db_options).ok()) { + } else if (ParseColumnFamilyOption( + o.first, o.second, &new_cf_options).ok()) { } else { return Status::InvalidArgument("Can't parse option " + o.first); } diff --git a/util/options_helper.h b/util/options_helper.h index c45f51d2f..ee885dd30 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -7,6 +7,8 @@ #include #include +#include + #include "rocksdb/options.h" #include "rocksdb/status.h" #include "rocksdb/table.h" @@ -118,6 +120,24 @@ struct OptionTypeInfo { bool SerializeSingleOptionHelper(const char* opt_address, const OptionType opt_type, std::string* value); +// In addition to its public version defined in rocksdb/convenience.h, +// this further takes an optional output vector "unsupported_options_names", +// which stores the name of all the unsupported options specified in "opts_map". +Status GetDBOptionsFromMapInternal( + const DBOptions& base_options, + const std::unordered_map& opts_map, + DBOptions* new_options, bool input_strings_escaped, + std::vector* unsupported_options_names = nullptr); + +// In addition to its public version defined in rocksdb/convenience.h, +// this further takes an optional output vector "unsupported_options_names", +// which stores the name of all the unsupported options specified in "opts_map". +Status GetColumnFamilyOptionsFromMapInternal( + const ColumnFamilyOptions& base_options, + const std::unordered_map& opts_map, + ColumnFamilyOptions* new_options, bool input_strings_escaped, + std::vector* unsupported_options_names = nullptr); + static std::unordered_map db_options_type_info = { /* // not yet supported diff --git a/util/options_parser.h b/util/options_parser.h index e5dfba17a..94e69cc2a 100644 --- a/util/options_parser.h +++ b/util/options_parser.h @@ -80,9 +80,9 @@ class RocksDBOptionsParser { const std::unordered_map* new_opt_map = nullptr, OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); - static Status VerifyTableFactory(const TableFactory* base_tf, - const TableFactory* file_tf, - OptionsSanityCheckLevel sanity_check_level); + static Status VerifyTableFactory( + const TableFactory* base_tf, const TableFactory* file_tf, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); static Status VerifyBlockBasedTableFactory( const BlockBasedTableFactory* base_tf, diff --git a/util/options_test.cc b/util/options_test.cc index 7940fde18..d1d4ecd21 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -16,14 +16,8 @@ #include #include "rocksdb/cache.h" -#include "rocksdb/compaction_filter.h" #include "rocksdb/convenience.h" -#include "rocksdb/merge_operator.h" -#include "rocksdb/options.h" -#include "rocksdb/table.h" #include "rocksdb/utilities/leveldb_options.h" -#include "table/block_based_table_factory.h" -#include "table/plain_table_factory.h" #include "util/options_helper.h" #include "util/options_parser.h" #include "util/options_sanity_check.h" @@ -75,163 +69,6 @@ Options PrintAndGetOptions(size_t total_write_buffer_limit, return options; } -class StringEnv : public EnvWrapper { - public: - class SeqStringSource : public SequentialFile { - public: - explicit SeqStringSource(const std::string& data) - : data_(data), offset_(0) {} - ~SeqStringSource() {} - Status Read(size_t n, Slice* result, char* scratch) override { - std::string output; - if (offset_ < data_.size()) { - n = std::min(data_.size() - offset_, n); - memcpy(scratch, data_.data() + offset_, n); - offset_ += n; - *result = Slice(scratch, n); - } else { - return Status::InvalidArgument( - "Attemp to read when it already reached eof."); - } - return Status::OK(); - } - Status Skip(uint64_t n) override { - if (offset_ >= data_.size()) { - return Status::InvalidArgument( - "Attemp to read when it already reached eof."); - } - // TODO(yhchiang): Currently doesn't handle the overflow case. - offset_ += n; - return Status::OK(); - } - - private: - std::string data_; - size_t offset_; - }; - - class StringSink : public WritableFile { - public: - explicit StringSink(std::string* contents) - : WritableFile(), contents_(contents) {} - virtual Status Truncate(uint64_t size) override { - contents_->resize(size); - return Status::OK(); - } - virtual Status Close() override { return Status::OK(); } - virtual Status Flush() override { return Status::OK(); } - virtual Status Sync() override { return Status::OK(); } - virtual Status Append(const Slice& slice) override { - contents_->append(slice.data(), slice.size()); - return Status::OK(); - } - - private: - std::string* contents_; - }; - - explicit StringEnv(Env* t) : EnvWrapper(t) {} - virtual ~StringEnv() {} - - const std::string& GetContent(const std::string& f) { return files_[f]; } - - const Status WriteToNewFile(const std::string& file_name, - const std::string& content) { - unique_ptr r; - auto s = NewWritableFile(file_name, &r, EnvOptions()); - if (!s.ok()) { - return s; - } - r->Append(content); - r->Flush(); - r->Close(); - assert(files_[file_name] == content); - return Status::OK(); - } - - // The following text is boilerplate that forwards all methods to target() - Status NewSequentialFile(const std::string& f, unique_ptr* r, - const EnvOptions& options) override { - auto iter = files_.find(f); - if (iter == files_.end()) { - return Status::NotFound("The specified file does not exist", f); - } - r->reset(new SeqStringSource(iter->second)); - return Status::OK(); - } - Status NewRandomAccessFile(const std::string& f, - unique_ptr* r, - const EnvOptions& options) override { - return Status::NotSupported(); - } - Status NewWritableFile(const std::string& f, unique_ptr* r, - const EnvOptions& options) override { - auto iter = files_.find(f); - if (iter != files_.end()) { - return Status::IOError("The specified file already exists", f); - } - r->reset(new StringSink(&files_[f])); - return Status::OK(); - } - virtual Status NewDirectory(const std::string& name, - unique_ptr* result) override { - return Status::NotSupported(); - } - Status FileExists(const std::string& f) override { - if (files_.find(f) == files_.end()) { - return Status::NotFound(); - } - return Status::OK(); - } - Status GetChildren(const std::string& dir, - std::vector* r) override { - return Status::NotSupported(); - } - Status DeleteFile(const std::string& f) override { - files_.erase(f); - return Status::OK(); - } - Status CreateDir(const std::string& d) override { - return Status::NotSupported(); - } - Status CreateDirIfMissing(const std::string& d) override { - return Status::NotSupported(); - } - Status DeleteDir(const std::string& d) override { - return Status::NotSupported(); - } - Status GetFileSize(const std::string& f, uint64_t* s) override { - auto iter = files_.find(f); - if (iter == files_.end()) { - return Status::NotFound("The specified file does not exist:", f); - } - *s = iter->second.size(); - return Status::OK(); - } - - Status GetFileModificationTime(const std::string& fname, - uint64_t* file_mtime) override { - return Status::NotSupported(); - } - - Status RenameFile(const std::string& s, const std::string& t) override { - return Status::NotSupported(); - } - - Status LinkFile(const std::string& s, const std::string& t) override { - return Status::NotSupported(); - } - - Status LockFile(const std::string& f, FileLock** l) override { - return Status::NotSupported(); - } - - Status UnlockFile(FileLock* l) override { return Status::NotSupported(); } - - protected: - std::unordered_map files_; -}; - class OptionsTest : public testing::Test {}; TEST_F(OptionsTest, LooseCondition) { @@ -736,70 +573,12 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) { ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); } -namespace { -void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { - // boolean options - db_opt->advise_random_on_open = rnd->Uniform(2); - db_opt->allow_mmap_reads = rnd->Uniform(2); - db_opt->allow_mmap_writes = rnd->Uniform(2); - db_opt->allow_os_buffer = rnd->Uniform(2); - db_opt->create_if_missing = rnd->Uniform(2); - db_opt->create_missing_column_families = rnd->Uniform(2); - db_opt->disableDataSync = rnd->Uniform(2); - db_opt->enable_thread_tracking = rnd->Uniform(2); - db_opt->error_if_exists = rnd->Uniform(2); - db_opt->is_fd_close_on_exec = rnd->Uniform(2); - db_opt->paranoid_checks = rnd->Uniform(2); - db_opt->skip_log_error_on_recovery = rnd->Uniform(2); - db_opt->skip_stats_update_on_db_open = rnd->Uniform(2); - db_opt->use_adaptive_mutex = rnd->Uniform(2); - db_opt->use_fsync = rnd->Uniform(2); - db_opt->recycle_log_file_num = rnd->Uniform(2); - - // int options - db_opt->max_background_compactions = rnd->Uniform(100); - db_opt->max_background_flushes = rnd->Uniform(100); - db_opt->max_file_opening_threads = rnd->Uniform(100); - db_opt->max_open_files = rnd->Uniform(100); - db_opt->table_cache_numshardbits = rnd->Uniform(100); - - // size_t options - db_opt->db_write_buffer_size = rnd->Uniform(10000); - db_opt->keep_log_file_num = rnd->Uniform(10000); - db_opt->log_file_time_to_roll = rnd->Uniform(10000); - db_opt->manifest_preallocation_size = rnd->Uniform(10000); - db_opt->max_log_file_size = rnd->Uniform(10000); - - // std::string options - db_opt->db_log_dir = "path/to/db_log_dir"; - db_opt->wal_dir = "path/to/wal_dir"; - - // uint32_t options - db_opt->max_subcompactions = rnd->Uniform(100000); - - // uint64_t options - static const uint64_t uint_max = static_cast(UINT_MAX); - db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000); - db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000); - db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000); - db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000); - db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000); - db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000); - db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000); - db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000); - - // unsigned int options - db_opt->stats_dump_period_sec = rnd->Uniform(100000); -} - -} // namespace - TEST_F(OptionsTest, DBOptionsSerialization) { Options base_options, new_options; Random rnd(301); // Phase 1: Make big change in base_options - RandomInitDBOptions(&base_options, &rnd); + test::RandomInitDBOptions(&base_options, &rnd); // Phase 2: obtain a string from base_option std::string base_options_file_content; @@ -812,212 +591,12 @@ TEST_F(OptionsTest, DBOptionsSerialization) { ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options)); } -namespace { -CompressionType RandomCompressionType(Random* rnd) { - return static_cast(rnd->Uniform(6)); -} - -void RandomCompressionTypeVector(const size_t count, - std::vector* types, - Random* rnd) { - types->clear(); - for (size_t i = 0; i < count; ++i) { - types->emplace_back(RandomCompressionType(rnd)); - } -} - -const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined = -1) { - int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4); - switch (random_num) { - case 0: - return NewFixedPrefixTransform(rnd->Uniform(20) + 1); - case 1: - return NewCappedPrefixTransform(rnd->Uniform(20) + 1); - case 2: - return NewNoopTransform(); - default: - return nullptr; - } -} - -TableFactory* RandomTableFactory(Random* rnd, int pre_defined = -1) { - int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(3); - switch (random_num) { - case 0: - return NewPlainTableFactory(); - case 1: - return NewCuckooTableFactory(); - default: - return NewBlockBasedTableFactory(); - } -} - -std::string RandomString(Random* rnd, const size_t len) { - std::stringstream ss; - for (size_t i = 0; i < len; ++i) { - ss << static_cast(rnd->Uniform(26) + 'a'); - } - return ss.str(); -} - -class ChanglingMergeOperator : public MergeOperator { - public: - explicit ChanglingMergeOperator(const std::string& name) - : name_(name + "MergeOperator") {} - ~ChanglingMergeOperator() {} - - void SetName(const std::string& name) { name_ = name; } - - virtual bool FullMerge(const Slice& key, const Slice* existing_value, - const std::deque& operand_list, - std::string* new_value, - Logger* logger) const override { - return false; - } - virtual bool PartialMergeMulti(const Slice& key, - const std::deque& operand_list, - std::string* new_value, - Logger* logger) const override { - return false; - } - virtual const char* Name() const override { return name_.c_str(); } - - protected: - std::string name_; -}; - -MergeOperator* RandomMergeOperator(Random* rnd) { - return new ChanglingMergeOperator(RandomString(rnd, 10)); -} - -class ChanglingCompactionFilter : public CompactionFilter { - public: - explicit ChanglingCompactionFilter(const std::string& name) - : name_(name + "CompactionFilter") {} - ~ChanglingCompactionFilter() {} - - void SetName(const std::string& name) { name_ = name; } - - bool Filter(int level, const Slice& key, const Slice& existing_value, - std::string* new_value, bool* value_changed) const override { - return false; - } - - const char* Name() const override { return name_.c_str(); } - - private: - std::string name_; -}; - -CompactionFilter* RandomCompactionFilter(Random* rnd) { - return new ChanglingCompactionFilter(RandomString(rnd, 10)); -} - -class ChanglingCompactionFilterFactory : public CompactionFilterFactory { - public: - explicit ChanglingCompactionFilterFactory(const std::string& name) - : name_(name + "CompactionFilterFactory") {} - ~ChanglingCompactionFilterFactory() {} - - void SetName(const std::string& name) { name_ = name; } - - std::unique_ptr CreateCompactionFilter( - const CompactionFilter::Context& context) override { - return std::unique_ptr(); - } - - // Returns a name that identifies this compaction filter factory. - const char* Name() const override { return name_.c_str(); } - - protected: - std::string name_; -}; - -CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) { - return new ChanglingCompactionFilterFactory(RandomString(rnd, 10)); -} - -// Note that the caller is responsible for releasing non-null -// cf_opt->compaction_filter. -void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) { - cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4)); - - // boolean options - cf_opt->compaction_measure_io_stats = rnd->Uniform(2); - cf_opt->disable_auto_compactions = rnd->Uniform(2); - cf_opt->filter_deletes = rnd->Uniform(2); - cf_opt->inplace_update_support = rnd->Uniform(2); - cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2); - cf_opt->optimize_filters_for_hits = rnd->Uniform(2); - cf_opt->paranoid_file_checks = rnd->Uniform(2); - cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2); - cf_opt->verify_checksums_in_compaction = rnd->Uniform(2); - - // double options - cf_opt->hard_rate_limit = static_cast(rnd->Uniform(10000)) / 13; - cf_opt->soft_rate_limit = static_cast(rnd->Uniform(10000)) / 13; - - // int options - cf_opt->expanded_compaction_factor = rnd->Uniform(100); - cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100); - cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100); - cf_opt->level0_stop_writes_trigger = rnd->Uniform(100); - cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100); - cf_opt->max_grandparent_overlap_factor = rnd->Uniform(100); - cf_opt->max_mem_compaction_level = rnd->Uniform(100); - cf_opt->max_write_buffer_number = rnd->Uniform(100); - cf_opt->max_write_buffer_number_to_maintain = rnd->Uniform(100); - cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100); - cf_opt->num_levels = rnd->Uniform(100); - cf_opt->source_compaction_factor = rnd->Uniform(100); - cf_opt->target_file_size_multiplier = rnd->Uniform(100); - - // size_t options - cf_opt->arena_block_size = rnd->Uniform(10000); - cf_opt->inplace_update_num_locks = rnd->Uniform(10000); - cf_opt->max_successive_merges = rnd->Uniform(10000); - cf_opt->memtable_prefix_bloom_huge_page_tlb_size = rnd->Uniform(10000); - cf_opt->write_buffer_size = rnd->Uniform(10000); - - // uint32_t options - cf_opt->bloom_locality = rnd->Uniform(10000); - cf_opt->memtable_prefix_bloom_bits = rnd->Uniform(10000); - cf_opt->memtable_prefix_bloom_probes = rnd->Uniform(10000); - cf_opt->min_partial_merge_operands = rnd->Uniform(10000); - cf_opt->max_bytes_for_level_base = rnd->Uniform(10000); - - // uint64_t options - static const uint64_t uint_max = static_cast(UINT_MAX); - cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000); - cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000); - - // unsigned int options - cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000); - - // pointer typed options - cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd)); - cf_opt->table_factory.reset(RandomTableFactory(rnd)); - cf_opt->merge_operator.reset(RandomMergeOperator(rnd)); - if (cf_opt->compaction_filter) { - delete cf_opt->compaction_filter; - } - cf_opt->compaction_filter = RandomCompactionFilter(rnd); - cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd)); - - // custom typed options - cf_opt->compression = RandomCompressionType(rnd); - RandomCompressionTypeVector(cf_opt->num_levels, - &cf_opt->compression_per_level, rnd); -} - -} // namespace - TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { ColumnFamilyOptions base_opt, new_opt; Random rnd(302); // Phase 1: randomly assign base_opt // custom type options - RandomInitCFOptions(&base_opt, &rnd); + test::RandomInitCFOptions(&base_opt, &rnd); // Phase 2: obtain a string from base_opt std::string base_options_file_content; @@ -1036,7 +615,6 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { #endif // !ROCKSDB_LITE - Status StringToMap( const std::string& opts_str, std::unordered_map* opts_map); @@ -1238,10 +816,10 @@ TEST_F(OptionsTest, ConvertOptionsTest) { #ifndef ROCKSDB_LITE class OptionsParserTest : public testing::Test { public: - OptionsParserTest() { env_.reset(new StringEnv(Env::Default())); } + OptionsParserTest() { env_.reset(new test::StringEnv(Env::Default())); } protected: - std::unique_ptr env_; + std::unique_ptr env_; }; TEST_F(OptionsParserTest, Comment) { @@ -1485,7 +1063,7 @@ void VerifyCFPointerTypedOptions( // change the name of merge operator back-and-forth { - auto* merge_operator = dynamic_cast( + auto* merge_operator = dynamic_cast( base_cf_opt->merge_operator.get()); if (merge_operator != nullptr) { name_buffer = merge_operator->Name(); @@ -1503,7 +1081,7 @@ void VerifyCFPointerTypedOptions( // change the name of the compaction filter factory back-and-forth { auto* compaction_filter_factory = - dynamic_cast( + dynamic_cast( base_cf_opt->compaction_filter_factory.get()); if (compaction_filter_factory != nullptr) { name_buffer = compaction_filter_factory->Name(); @@ -1573,17 +1151,17 @@ TEST_F(OptionsParserTest, DumpAndParse) { "###rocksdb#1-testcf#2###"}; const int num_cf = static_cast(cf_names.size()); Random rnd(302); - RandomInitDBOptions(&base_db_opt, &rnd); + test::RandomInitDBOptions(&base_db_opt, &rnd); base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG"; for (int c = 0; c < num_cf; ++c) { ColumnFamilyOptions cf_opt; Random cf_rnd(0xFB + c); - RandomInitCFOptions(&cf_opt, &cf_rnd); + test::RandomInitCFOptions(&cf_opt, &cf_rnd); if (c < 4) { - cf_opt.prefix_extractor.reset(RandomSliceTransform(&rnd, c)); + cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c)); } if (c < 3) { - cf_opt.table_factory.reset(RandomTableFactory(&rnd, c)); + cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c)); } base_cf_opts.emplace_back(cf_opt); } @@ -1717,7 +1295,7 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { { for (int tb = 2; tb >= 0; --tb) { // change the table factory - opts.table_factory.reset(RandomTableFactory(&rnd, tb)); + opts.table_factory.reset(test::RandomTableFactory(&rnd, tb)); ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); @@ -1731,7 +1309,7 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { { for (int test = 0; test < 5; ++test) { // change the merge operator - opts.merge_operator.reset(RandomMergeOperator(&rnd)); + opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); @@ -1745,7 +1323,7 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { { for (int test = 0; test < 5; ++test) { // change the compaction filter - opts.compaction_filter = RandomCompactionFilter(&rnd); + opts.compaction_filter = test::RandomCompactionFilter(&rnd); ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); @@ -1761,7 +1339,8 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) { { for (int test = 0; test < 5; ++test) { // change the compaction filter factory - opts.compaction_filter_factory.reset(RandomCompactionFilterFactory(&rnd)); + opts.compaction_filter_factory.reset( + test::RandomCompactionFilterFactory(&rnd)); ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); diff --git a/util/testutil.cc b/util/testutil.cc index b995a2e53..8db8dac88 100644 --- a/util/testutil.cc +++ b/util/testutil.cc @@ -9,9 +9,11 @@ #include "util/testutil.h" +#include +#include + #include "port/port.h" #include "util/file_reader_writer.h" -#include "util/random.h" namespace rocksdb { namespace test { @@ -153,5 +155,209 @@ std::string KeyStr(const std::string& user_key, const SequenceNumber& seq, return k.Encode().ToString(); } +std::string RandomName(Random* rnd, const size_t len) { + std::stringstream ss; + for (size_t i = 0; i < len; ++i) { + ss << static_cast(rnd->Uniform(26) + 'a'); + } + return ss.str(); +} + +CompressionType RandomCompressionType(Random* rnd) { + return static_cast(rnd->Uniform(6)); +} + +void RandomCompressionTypeVector(const size_t count, + std::vector* types, + Random* rnd) { + types->clear(); + for (size_t i = 0; i < count; ++i) { + types->emplace_back(RandomCompressionType(rnd)); + } +} + +const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined) { + int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4); + switch (random_num) { + case 0: + return NewFixedPrefixTransform(rnd->Uniform(20) + 1); + case 1: + return NewCappedPrefixTransform(rnd->Uniform(20) + 1); + case 2: + return NewNoopTransform(); + default: + return nullptr; + } +} + +BlockBasedTableOptions RandomBlockBasedTableOptions(Random* rnd) { + BlockBasedTableOptions opt; + opt.cache_index_and_filter_blocks = rnd->Uniform(2); + opt.index_type = rnd->Uniform(2) ? BlockBasedTableOptions::kBinarySearch + : BlockBasedTableOptions::kHashSearch; + opt.hash_index_allow_collision = rnd->Uniform(2); + opt.checksum = static_cast(rnd->Uniform(3)); + opt.block_size = rnd->Uniform(10000000); + opt.block_size_deviation = rnd->Uniform(100); + opt.block_restart_interval = rnd->Uniform(100); + opt.whole_key_filtering = rnd->Uniform(2); + + return opt; +} + +TableFactory* RandomTableFactory(Random* rnd, int pre_defined) { +#ifndef ROCKSDB_LITE + int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4); + switch (random_num) { + case 0: + return NewPlainTableFactory(); + case 1: + return NewCuckooTableFactory(); + default: + return NewBlockBasedTableFactory(); + } +#else + return NewBlockBasedTableFactory(); +#endif // !ROCKSDB_LITE +} + +MergeOperator* RandomMergeOperator(Random* rnd) { + return new ChanglingMergeOperator(RandomName(rnd, 10)); +} + +CompactionFilter* RandomCompactionFilter(Random* rnd) { + return new ChanglingCompactionFilter(RandomName(rnd, 10)); +} + +CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) { + return new ChanglingCompactionFilterFactory(RandomName(rnd, 10)); +} + +void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { + // boolean options + db_opt->advise_random_on_open = rnd->Uniform(2); + db_opt->allow_mmap_reads = rnd->Uniform(2); + db_opt->allow_mmap_writes = rnd->Uniform(2); + db_opt->allow_os_buffer = rnd->Uniform(2); + db_opt->create_if_missing = rnd->Uniform(2); + db_opt->create_missing_column_families = rnd->Uniform(2); + db_opt->disableDataSync = rnd->Uniform(2); + db_opt->enable_thread_tracking = rnd->Uniform(2); + db_opt->error_if_exists = rnd->Uniform(2); + db_opt->is_fd_close_on_exec = rnd->Uniform(2); + db_opt->paranoid_checks = rnd->Uniform(2); + db_opt->skip_log_error_on_recovery = rnd->Uniform(2); + db_opt->skip_stats_update_on_db_open = rnd->Uniform(2); + db_opt->use_adaptive_mutex = rnd->Uniform(2); + db_opt->use_fsync = rnd->Uniform(2); + db_opt->recycle_log_file_num = rnd->Uniform(2); + + // int options + db_opt->max_background_compactions = rnd->Uniform(100); + db_opt->max_background_flushes = rnd->Uniform(100); + db_opt->max_file_opening_threads = rnd->Uniform(100); + db_opt->max_open_files = rnd->Uniform(100); + db_opt->table_cache_numshardbits = rnd->Uniform(100); + + // size_t options + db_opt->db_write_buffer_size = rnd->Uniform(10000); + db_opt->keep_log_file_num = rnd->Uniform(10000); + db_opt->log_file_time_to_roll = rnd->Uniform(10000); + db_opt->manifest_preallocation_size = rnd->Uniform(10000); + db_opt->max_log_file_size = rnd->Uniform(10000); + + // std::string options + db_opt->db_log_dir = "path/to/db_log_dir"; + db_opt->wal_dir = "path/to/wal_dir"; + + // uint32_t options + db_opt->max_subcompactions = rnd->Uniform(100000); + + // uint64_t options + static const uint64_t uint_max = static_cast(UINT_MAX); + db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000); + db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000); + db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000); + db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000); + db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000); + db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000); + db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000); + db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000); + + // unsigned int options + db_opt->stats_dump_period_sec = rnd->Uniform(100000); +} + +void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) { + cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4)); + + // boolean options + cf_opt->compaction_measure_io_stats = rnd->Uniform(2); + cf_opt->disable_auto_compactions = rnd->Uniform(2); + cf_opt->filter_deletes = rnd->Uniform(2); + cf_opt->inplace_update_support = rnd->Uniform(2); + cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2); + cf_opt->optimize_filters_for_hits = rnd->Uniform(2); + cf_opt->paranoid_file_checks = rnd->Uniform(2); + cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2); + cf_opt->verify_checksums_in_compaction = rnd->Uniform(2); + + // double options + cf_opt->hard_rate_limit = static_cast(rnd->Uniform(10000)) / 13; + cf_opt->soft_rate_limit = static_cast(rnd->Uniform(10000)) / 13; + + // int options + cf_opt->expanded_compaction_factor = rnd->Uniform(100); + cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100); + cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100); + cf_opt->level0_stop_writes_trigger = rnd->Uniform(100); + cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100); + cf_opt->max_grandparent_overlap_factor = rnd->Uniform(100); + cf_opt->max_mem_compaction_level = rnd->Uniform(100); + cf_opt->max_write_buffer_number = rnd->Uniform(100); + cf_opt->max_write_buffer_number_to_maintain = rnd->Uniform(100); + cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100); + cf_opt->num_levels = rnd->Uniform(100); + cf_opt->source_compaction_factor = rnd->Uniform(100); + cf_opt->target_file_size_multiplier = rnd->Uniform(100); + + // size_t options + cf_opt->arena_block_size = rnd->Uniform(10000); + cf_opt->inplace_update_num_locks = rnd->Uniform(10000); + cf_opt->max_successive_merges = rnd->Uniform(10000); + cf_opt->memtable_prefix_bloom_huge_page_tlb_size = rnd->Uniform(10000); + cf_opt->write_buffer_size = rnd->Uniform(10000); + + // uint32_t options + cf_opt->bloom_locality = rnd->Uniform(10000); + cf_opt->memtable_prefix_bloom_bits = rnd->Uniform(10000); + cf_opt->memtable_prefix_bloom_probes = rnd->Uniform(10000); + cf_opt->min_partial_merge_operands = rnd->Uniform(10000); + cf_opt->max_bytes_for_level_base = rnd->Uniform(10000); + + // uint64_t options + static const uint64_t uint_max = static_cast(UINT_MAX); + cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000); + cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000); + + // unsigned int options + cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000); + + // pointer typed options + cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd)); + cf_opt->table_factory.reset(RandomTableFactory(rnd)); + cf_opt->merge_operator.reset(RandomMergeOperator(rnd)); + if (cf_opt->compaction_filter) { + delete cf_opt->compaction_filter; + } + cf_opt->compaction_filter = RandomCompactionFilter(rnd); + cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd)); + + // custom typed options + cf_opt->compression = RandomCompressionType(rnd); + RandomCompressionTypeVector(cf_opt->num_levels, + &cf_opt->compression_per_level, rnd); +} + } // namespace test } // namespace rocksdb diff --git a/util/testutil.h b/util/testutil.h index 0373532a8..a47d5abce 100644 --- a/util/testutil.h +++ b/util/testutil.h @@ -9,6 +9,7 @@ #pragma once #include +#include #include #include @@ -16,8 +17,13 @@ #include "rocksdb/compaction_filter.h" #include "rocksdb/env.h" #include "rocksdb/iterator.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" #include "rocksdb/slice.h" +#include "rocksdb/table.h" +#include "table/block_based_table_factory.h" #include "table/internal_iterator.h" +#include "table/plain_table_factory.h" #include "util/mutexlock.h" #include "util/random.h" @@ -378,5 +384,258 @@ inline std::string EncodeInt(uint64_t x) { return result; } +class StringEnv : public EnvWrapper { + public: + class SeqStringSource : public SequentialFile { + public: + explicit SeqStringSource(const std::string& data) + : data_(data), offset_(0) {} + ~SeqStringSource() {} + Status Read(size_t n, Slice* result, char* scratch) override { + std::string output; + if (offset_ < data_.size()) { + n = std::min(data_.size() - offset_, n); + memcpy(scratch, data_.data() + offset_, n); + offset_ += n; + *result = Slice(scratch, n); + } else { + return Status::InvalidArgument( + "Attemp to read when it already reached eof."); + } + return Status::OK(); + } + Status Skip(uint64_t n) override { + if (offset_ >= data_.size()) { + return Status::InvalidArgument( + "Attemp to read when it already reached eof."); + } + // TODO(yhchiang): Currently doesn't handle the overflow case. + offset_ += n; + return Status::OK(); + } + + private: + std::string data_; + size_t offset_; + }; + + class StringSink : public WritableFile { + public: + explicit StringSink(std::string* contents) + : WritableFile(), contents_(contents) {} + virtual Status Truncate(uint64_t size) override { + contents_->resize(size); + return Status::OK(); + } + virtual Status Close() override { return Status::OK(); } + virtual Status Flush() override { return Status::OK(); } + virtual Status Sync() override { return Status::OK(); } + virtual Status Append(const Slice& slice) override { + contents_->append(slice.data(), slice.size()); + return Status::OK(); + } + + private: + std::string* contents_; + }; + + explicit StringEnv(Env* t) : EnvWrapper(t) {} + virtual ~StringEnv() {} + + const std::string& GetContent(const std::string& f) { return files_[f]; } + + const Status WriteToNewFile(const std::string& file_name, + const std::string& content) { + unique_ptr r; + auto s = NewWritableFile(file_name, &r, EnvOptions()); + if (!s.ok()) { + return s; + } + r->Append(content); + r->Flush(); + r->Close(); + assert(files_[file_name] == content); + return Status::OK(); + } + + // The following text is boilerplate that forwards all methods to target() + Status NewSequentialFile(const std::string& f, unique_ptr* r, + const EnvOptions& options) override { + auto iter = files_.find(f); + if (iter == files_.end()) { + return Status::NotFound("The specified file does not exist", f); + } + r->reset(new SeqStringSource(iter->second)); + return Status::OK(); + } + Status NewRandomAccessFile(const std::string& f, + unique_ptr* r, + const EnvOptions& options) override { + return Status::NotSupported(); + } + Status NewWritableFile(const std::string& f, unique_ptr* r, + const EnvOptions& options) override { + auto iter = files_.find(f); + if (iter != files_.end()) { + return Status::IOError("The specified file already exists", f); + } + r->reset(new StringSink(&files_[f])); + return Status::OK(); + } + virtual Status NewDirectory(const std::string& name, + unique_ptr* result) override { + return Status::NotSupported(); + } + Status FileExists(const std::string& f) override { + if (files_.find(f) == files_.end()) { + return Status::NotFound(); + } + return Status::OK(); + } + Status GetChildren(const std::string& dir, + std::vector* r) override { + return Status::NotSupported(); + } + Status DeleteFile(const std::string& f) override { + files_.erase(f); + return Status::OK(); + } + Status CreateDir(const std::string& d) override { + return Status::NotSupported(); + } + Status CreateDirIfMissing(const std::string& d) override { + return Status::NotSupported(); + } + Status DeleteDir(const std::string& d) override { + return Status::NotSupported(); + } + Status GetFileSize(const std::string& f, uint64_t* s) override { + auto iter = files_.find(f); + if (iter == files_.end()) { + return Status::NotFound("The specified file does not exist:", f); + } + *s = iter->second.size(); + return Status::OK(); + } + + Status GetFileModificationTime(const std::string& fname, + uint64_t* file_mtime) override { + return Status::NotSupported(); + } + + Status RenameFile(const std::string& s, const std::string& t) override { + return Status::NotSupported(); + } + + Status LinkFile(const std::string& s, const std::string& t) override { + return Status::NotSupported(); + } + + Status LockFile(const std::string& f, FileLock** l) override { + return Status::NotSupported(); + } + + Status UnlockFile(FileLock* l) override { return Status::NotSupported(); } + + protected: + std::unordered_map files_; +}; + +// Randomly initialize the given DBOptions +void RandomInitDBOptions(DBOptions* db_opt, Random* rnd); + +// Randomly initialize the given ColumnFamilyOptions +// Note that the caller is responsible for releasing non-null +// cf_opt->compaction_filter. +void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd); + +// A dummy merge operator which can change its name +class ChanglingMergeOperator : public MergeOperator { + public: + explicit ChanglingMergeOperator(const std::string& name) + : name_(name + "MergeOperator") {} + ~ChanglingMergeOperator() {} + + void SetName(const std::string& name) { name_ = name; } + + virtual bool FullMerge(const Slice& key, const Slice* existing_value, + const std::deque& operand_list, + std::string* new_value, + Logger* logger) const override { + return false; + } + virtual bool PartialMergeMulti(const Slice& key, + const std::deque& operand_list, + std::string* new_value, + Logger* logger) const override { + return false; + } + virtual const char* Name() const override { return name_.c_str(); } + + protected: + std::string name_; +}; + +// Returns a dummy merge operator with random name. +MergeOperator* RandomMergeOperator(Random* rnd); + +// A dummy compaction filter which can change its name +class ChanglingCompactionFilter : public CompactionFilter { + public: + explicit ChanglingCompactionFilter(const std::string& name) + : name_(name + "CompactionFilter") {} + ~ChanglingCompactionFilter() {} + + void SetName(const std::string& name) { name_ = name; } + + bool Filter(int level, const Slice& key, const Slice& existing_value, + std::string* new_value, bool* value_changed) const override { + return false; + } + + const char* Name() const override { return name_.c_str(); } + + private: + std::string name_; +}; + +// Returns a dummy compaction filter with a random name. +CompactionFilter* RandomCompactionFilter(Random* rnd); + +// A dummy compaction filter factory which can change its name +class ChanglingCompactionFilterFactory : public CompactionFilterFactory { + public: + explicit ChanglingCompactionFilterFactory(const std::string& name) + : name_(name + "CompactionFilterFactory") {} + ~ChanglingCompactionFilterFactory() {} + + void SetName(const std::string& name) { name_ = name; } + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + return std::unique_ptr(); + } + + // Returns a name that identifies this compaction filter factory. + const char* Name() const override { return name_.c_str(); } + + protected: + std::string name_; +}; + +CompressionType RandomCompressionType(Random* rnd); + +void RandomCompressionTypeVector(const size_t count, + std::vector* types, + Random* rnd); + +CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd); + +const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined = -1); + +TableFactory* RandomTableFactory(Random* rnd, int pre_defined = -1); + +std::string RandomName(Random* rnd, const size_t len); + } // namespace test } // namespace rocksdb diff --git a/utilities/options/options_util.cc b/utilities/options/options_util.cc new file mode 100644 index 000000000..6a595c3d8 --- /dev/null +++ b/utilities/options/options_util.cc @@ -0,0 +1,76 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef ROCKSDB_LITE + +#include "rocksdb/utilities/options_util.h" + +#include "db/filename.h" +#include "rocksdb/options.h" +#include "util/options_parser.h" + +namespace rocksdb { +Status LoadOptionsFromFile(const std::string& file_name, Env* env, + DBOptions* db_options, + std::vector* cf_descs) { + RocksDBOptionsParser parser; + Status s = parser.Parse(file_name, env); + if (!s.ok()) { + return s; + } + + *db_options = *parser.db_opt(); + + const std::vector& cf_names = *parser.cf_names(); + const std::vector& cf_opts = *parser.cf_opts(); + cf_descs->clear(); + for (size_t i = 0; i < cf_opts.size(); ++i) { + cf_descs->push_back({cf_names[i], cf_opts[i]}); + } + return Status::OK(); +} + +Status GetLatestOptionsFileName(const std::string& dbpath, + Env* env, std::string* options_file_name) { + Status s; + std::string latest_file_name; + uint64_t latest_time_stamp = 0; + std::vector file_names; + s = env->GetChildren(dbpath, &file_names); + if (!s.ok()) { + return s; + } + for (auto& file_name : file_names) { + uint64_t time_stamp; + FileType type; + if (ParseFileName(file_name, &time_stamp, &type) && type == kOptionsFile) { + if (time_stamp > latest_time_stamp) { + latest_time_stamp = time_stamp; + latest_file_name = file_name; + } + } + } + if (latest_file_name.size() == 0) { + return Status::NotFound("No options files found in the DB directory."); + } + *options_file_name = latest_file_name; + return Status::OK(); +} + +Status LoadLatestOptions(const std::string& dbpath, Env* env, + DBOptions* db_options, + std::vector* cf_descs) { + std::string options_file_name; + Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); + if (!s.ok()) { + return s; + } + + return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, + db_options, cf_descs); +} + +} // namespace rocksdb +#endif // !ROCKSDB_LITE diff --git a/utilities/options/options_util_test.cc b/utilities/options/options_util_test.cc new file mode 100644 index 000000000..0ad29fb4b --- /dev/null +++ b/utilities/options/options_util_test.cc @@ -0,0 +1,102 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef ROCKSDB_LITE +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include + +#include +#include + +#include "rocksdb/utilities/options_util.h" +#include "util/options_parser.h" +#include "util/random.h" +#include "util/testharness.h" +#include "util/testutil.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include +using GFLAGS::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace rocksdb { +class OptionsUtilTest : public testing::Test { + public: + OptionsUtilTest() { env_.reset(new test::StringEnv(Env::Default())); } + + protected: + std::unique_ptr env_; +}; + +bool IsBlockBasedTableFactory(TableFactory* tf) { + return tf->Name() == BlockBasedTableFactory().Name(); +} + +TEST_F(OptionsUtilTest, SaveAndLoad) { + const size_t kCFCount = 5; + Random rnd(0xFB); + + DBOptions db_opt; + std::vector cf_names; + std::vector cf_opts; + test::RandomInitDBOptions(&db_opt, &rnd); + for (size_t i = 0; i < kCFCount; ++i) { + cf_names.push_back(i == 0 ? kDefaultColumnFamilyName + : test::RandomName(&rnd, 10)); + cf_opts.emplace_back(); + test::RandomInitCFOptions(&cf_opts.back(), &rnd); + } + + const std::string kFileName = "OPTIONS-123456"; + PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, env_.get()); + + DBOptions loaded_db_opt; + std::vector loaded_cf_descs; + ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, + &loaded_cf_descs)); + + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); + test::RandomInitDBOptions(&db_opt, &rnd); + ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); + + for (size_t i = 0; i < kCFCount; ++i) { + ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + cf_opts[i], loaded_cf_descs[i].options)); + if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { + ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory( + cf_opts[i].table_factory.get(), + loaded_cf_descs[i].options.table_factory.get())); + } + test::RandomInitCFOptions(&cf_opts[i], &rnd); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + cf_opts[i], loaded_cf_descs[i].options)); + } +} + +} // namespace rocksdb + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} + +#else +#include + +int main(int argc, char** argv) { + printf("Skipped in RocksDBLite as utilities are not supported.\n"); + return 0; +} +#endif // !ROCKSDB_LITE