From c442f6809f3913641ca4bed50548159974438e7f Mon Sep 17 00:00:00 2001 From: mrambacher Date: Wed, 11 Nov 2020 15:09:14 -0800 Subject: [PATCH] Create a Customizable class to load classes and configurations (#6590) Summary: The Customizable class is an extension of the Configurable class and allows instances to be created by a name/ID. Classes that extend customizable can define their Type (e.g. "TableFactory", "Cache") and a method to instantiate them (TableFactory::CreateFromString). Customizable objects can be registered with the ObjectRegistry and created dynamically. Future PRs will make more types of objects extend Customizable. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6590 Reviewed By: cheng-chang Differential Revision: D24841553 Pulled By: zhichao-cao fbshipit-source-id: d0c2132bd932e971cbfe2c908ca2e5db30c5e155 --- CMakeLists.txt | 2 + HISTORY.md | 1 + Makefile | 4 + TARGETS | 9 + include/rocksdb/configurable.h | 5 - include/rocksdb/customizable.h | 138 ++++ include/rocksdb/table.h | 20 +- include/rocksdb/utilities/options_type.h | 133 +++- options/cf_options.cc | 32 +- options/configurable.cc | 107 ++- options/configurable_helper.h | 40 ++ options/configurable_test.cc | 38 -- options/customizable.cc | 77 +++ options/customizable_helper.h | 216 ++++++ options/customizable_test.cc | 625 ++++++++++++++++++ options/options_helper.cc | 13 + src.mk | 2 + table/block_based/block_based_table_factory.h | 3 + table/cuckoo/cuckoo_table_factory.h | 2 + table/plain/plain_table_factory.h | 2 + table/table_factory.cc | 36 +- 21 files changed, 1369 insertions(+), 136 deletions(-) create mode 100644 include/rocksdb/customizable.h create mode 100644 options/customizable.cc create mode 100644 options/customizable_helper.h create mode 100644 options/customizable_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index c9f36ada1..fdc894734 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -687,6 +687,7 @@ set(SOURCES monitoring/thread_status_util_debug.cc options/cf_options.cc options/configurable.cc + options/customizable.cc options/db_options.cc options/options.cc options/options_helper.cc @@ -1142,6 +1143,7 @@ if(WITH_TESTS) monitoring/statistics_test.cc monitoring/stats_history_test.cc options/configurable_test.cc + options/customizable_test.cc options/options_settable_test.cc options/options_test.cc table/block_based/block_based_filter_block_test.cc diff --git a/HISTORY.md b/HISTORY.md index cca59608f..58c178ac3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -42,6 +42,7 @@ * Added is_full_compaction to CompactionJobStats, so that the information is available through the EventListener interface. * Add more stats for MultiGet in Histogram to get number of data blocks, index blocks, filter blocks and sst files read from file system per level. * SST files have a new table property called db_host_id, which is set to the hostname by default. A new option in DBOptions, db_host_id, allows the property value to be overridden with a user specified string, or disable it completely by making the option string empty. +* Methods to create customizable extensions -- such as TableFactory -- are exposed directly through the Customizable base class (from which these objects inherit). This change will allow these Customizable classes to be loaded and configured in a standard way (via CreateFromString). More information on how to write and use Customizable classes is in the customizable.h header file. ## 6.13 (09/12/2020) ### Bug fixes diff --git a/Makefile b/Makefile index 37ffa032e..54cfc9e97 100644 --- a/Makefile +++ b/Makefile @@ -628,6 +628,7 @@ ifdef ASSERT_STATUS_CHECKED plain_table_db_test \ repair_test \ configurable_test \ + customizable_test \ options_settable_test \ options_test \ random_test \ @@ -1800,6 +1801,9 @@ compact_files_test: $(OBJ_DIR)/db/compact_files_test.o $(TEST_LIBRARY) $(LIBRARY configurable_test: options/configurable_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +customizable_test: options/customizable_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + options_test: $(OBJ_DIR)/options/options_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) diff --git a/TARGETS b/TARGETS index 83260b24a..ba2287d6a 100644 --- a/TARGETS +++ b/TARGETS @@ -255,6 +255,7 @@ cpp_library( "monitoring/thread_status_util_debug.cc", "options/cf_options.cc", "options/configurable.cc", + "options/customizable.cc", "options/db_options.cc", "options/options.cc", "options/options_helper.cc", @@ -544,6 +545,7 @@ cpp_library( "monitoring/thread_status_util_debug.cc", "options/cf_options.cc", "options/configurable.cc", + "options/customizable.cc", "options/db_options.cc", "options/options.cc", "options/options_helper.cc", @@ -1090,6 +1092,13 @@ ROCKS_TESTS = [ [], [], ], + [ + "customizable_test", + "options/customizable_test.cc", + "serial", + [], + [], + ], [ "data_block_hash_index_test", "table/block_based/data_block_hash_index_test.cc", diff --git a/include/rocksdb/configurable.h b/include/rocksdb/configurable.h index f4bfbf532..95c5cf4d6 100644 --- a/include/rocksdb/configurable.h +++ b/include/rocksdb/configurable.h @@ -270,11 +270,6 @@ class Configurable { // True once the object is prepared. Once the object is prepared, only // mutable options can be configured. bool prepared_; - // If this class is a wrapper (has-a), this method should be - // over-written to return the inner configurable (like an EnvWrapper). - // This method should NOT recurse, but should instead return the - // direct Inner object. - virtual Configurable* Inner() const { return nullptr; } // Returns the raw pointer for the associated named option. // The name is typically the name of an option registered via the diff --git a/include/rocksdb/customizable.h b/include/rocksdb/customizable.h new file mode 100644 index 000000000..366c7563f --- /dev/null +++ b/include/rocksdb/customizable.h @@ -0,0 +1,138 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "rocksdb/configurable.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +/** + * Customizable a base class used by the rocksdb that describes a + * standard way of configuring and creating objects. Customizable objects + * are configurable objects that can be created from an ObjectRegistry. + * + * Customizable classes are used when there are multiple potential + * implementations of a class for use by RocksDB (e.g. Table, Cache, + * MergeOperator, etc). The abstract base class is expected to define a method + * declaring its type and a factory method for creating one of these, such as: + * static const char *Type() { return "Table"; } + * static Status CreateFromString(const ConfigOptions& options, + * const std::string& id, + * std::shared_ptr* result); + * The "Type" string is expected to be unique (no two base classes are the same + * type). This factory is expected, based on the options and id, create and + * return the appropriate derived type of the customizable class (e.g. + * BlockBasedTableFactory, PlainTableFactory, etc). For extension developers, + * helper classes and methods are provided for writing this factory. + * + * Instances of a Customizable class need to define: + * - A "static const char *kClassName()" method. This method defines the name + * of the class instance (e.g. BlockBasedTable, LRUCache) and is used by the + * CheckedCast method. + * - The Name() of the object. This name is used when creating and saving + * instances of this class. Typically this name will be the same as + * kClassName(). + * + * Additionally, Customizable classes should register any options used to + * configure themselves with the Configurable subsystem. + * + * When a Customizable is being created, the "name" property specifies + * the name of the instance being created. + * For custom objects, their configuration and name can be specified by: + * [prop]={name=X;option 1 = value1[; option2=value2...]} + * + * [prop].name=X + * [prop].option1 = value1 + * + * [prop].name=X + * X.option1 =value1 + */ +class Customizable : public Configurable { + public: + virtual ~Customizable() {} + + // Returns the name of this class of Customizable + virtual const char* Name() const = 0; + + // Returns an identifier for this Customizable. + // This could be its name or something more complex (like its URL/pattern). + // Used for pretty printing. + virtual std::string GetId() const { + std::string id = Name(); + return id; + } + + // This is typically determined by if the input name matches the + // name of this object. + // This method is typically used in conjunction with CheckedCast to find the + // derived class instance from its base. For example, if you have an Env + // and want the "Default" env, you would IsInstanceOf("Default") to get + // the default implementation. This method should be used when you need a + // specific derivative or implementation of a class. + // + // Intermediary caches (such as SharedCache) may wish to override this method + // to check for the intermediary name (SharedCache). Classes with multiple + // potential names (e.g. "PosixEnv", "DefaultEnv") may also wish to override + // this method. + // + // @param name The name of the instance to find. + // Returns true if the class is an instance of the input name. + virtual bool IsInstanceOf(const std::string& name) const { + return name == Name(); + } + + // Returns the named instance of the Customizable as a T*, or nullptr if not + // found. This method uses IsInstanceOf to find the appropriate class instance + // and then casts it to the expected return type. + template + const T* CheckedCast() const { + if (IsInstanceOf(T::kClassName())) { + return static_cast(this); + } else { + return nullptr; + } + } + + template + T* CheckedCast() { + if (IsInstanceOf(T::kClassName())) { + return static_cast(this); + } else { + return nullptr; + } + } + + // Checks to see if this Customizable is equivalent to other. + // This method assumes that the two objects are of the same class. + // @param config_options Controls how the options are compared. + // @param other The other object to compare to. + // @param mismatch If the objects do not match, this parameter contains + // the name of the option that triggered the match failure. + // @param True if the objects match, false otherwise. + // @see Configurable::AreEquivalent for more details + bool AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* mismatch) const override; +#ifndef ROCKSDB_LITE + // Gets the value of the option associated with the input name + // @see Configurable::GetOption for more details + Status GetOption(const ConfigOptions& config_options, const std::string& name, + std::string* value) const override; + +#endif // ROCKSDB_LITE + protected: + // Given a name (e.g. rocksdb.my.type.opt), returns the short name (opt) + std::string GetOptionName(const std::string& long_name) const override; +#ifndef ROCKSDB_LITE + std::string SerializeOptions(const ConfigOptions& options, + const std::string& prefix) const override; +#endif // ROCKSDB_LITE +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 1e779e39a..a2bfe3cb4 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -22,7 +22,7 @@ #include #include -#include "rocksdb/configurable.h" +#include "rocksdb/customizable.h" #include "rocksdb/env.h" #include "rocksdb/options.h" #include "rocksdb/status.h" @@ -613,7 +613,7 @@ extern TableFactory* NewCuckooTableFactory( class RandomAccessFileReader; // A base class for table factories. -class TableFactory : public Configurable { +class TableFactory : public Customizable { public: virtual ~TableFactory() override {} @@ -627,21 +627,7 @@ class TableFactory : public Configurable { const std::string& id, std::shared_ptr* factory); - // The type of the table. - // - // The client of this package should switch to a new name whenever - // the table format implementation changes. - // - // Names starting with "rocksdb." are reserved and should not be used - // by any clients of this package. - virtual const char* Name() const = 0; - - // Returns true if the class is an instance of the input name. - // This is typically determined by if the input name matches the - // name of this object. - virtual bool IsInstanceOf(const std::string& name) const { - return name == Name(); - } + static const char* Type() { return "TableFactory"; } // Returns a Table object table that can fetch data from file specified // in parameter file. It's the caller's responsibility to make sure diff --git a/include/rocksdb/utilities/options_type.h b/include/rocksdb/utilities/options_type.h index 2bd081abf..36e1e09f9 100644 --- a/include/rocksdb/utilities/options_type.h +++ b/include/rocksdb/utilities/options_type.h @@ -49,6 +49,7 @@ enum class OptionType { kStruct, kVector, kConfigurable, + kCustomizable, kUnknown, }; @@ -93,13 +94,14 @@ enum class OptionTypeFlags : uint32_t { kCompareLoose = ConfigOptions::kSanityLevelLooselyCompatible, kCompareExact = ConfigOptions::kSanityLevelExactMatch, - kMutable = 0x0100, // Option is mutable - kRawPointer = 0x0200, // The option is stored as a raw pointer - kShared = 0x0400, // The option is stored as a shared_ptr - kUnique = 0x0800, // The option is stored as a unique_ptr - kAllowNull = 0x1000, // The option can be null - kDontSerialize = 0x2000, // Don't serialize the option - kDontPrepare = 0x4000, // Don't prepare or sanitize this option + kMutable = 0x0100, // Option is mutable + kRawPointer = 0x0200, // The option is stored as a raw pointer + kShared = 0x0400, // The option is stored as a shared_ptr + kUnique = 0x0800, // The option is stored as a unique_ptr + kAllowNull = 0x1000, // The option can be null + kDontSerialize = 0x2000, // Don't serialize the option + kDontPrepare = 0x4000, // Don't prepare or sanitize this option + kStringNameOnly = 0x8000, // The option serializes to a name only }; inline OptionTypeFlags operator|(const OptionTypeFlags &a, @@ -406,6 +408,103 @@ class OptionTypeInfo { }); } + // Create a new std::shared_ptr OptionTypeInfo + // This function will call the T::CreateFromString method to create a new + // std::shared_ptr object. + // + // @param offset The offset for the Customizable from the base pointer + // @param ovt How to verify this option + // @param flags, Extra flags specifying the behavior of this option + // @param _sfunc Optional function for serializing this option + // @param _efunc Optional function for comparing this option + template + static OptionTypeInfo AsCustomSharedPtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags) { + return AsCustomSharedPtr(offset, ovt, flags, nullptr, nullptr); + } + + template + static OptionTypeInfo AsCustomSharedPtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags, + const SerializeFunc& serialize_func, + const EqualsFunc& equals_func) { + return OptionTypeInfo( + offset, OptionType::kCustomizable, ovt, + flags | OptionTypeFlags::kShared, + [](const ConfigOptions& opts, const std::string&, + const std::string& value, char* addr) { + auto* shared = reinterpret_cast*>(addr); + return T::CreateFromString(opts, value, shared); + }, + serialize_func, equals_func); + } + + // Create a new std::unique_ptr OptionTypeInfo + // This function will call the T::CreateFromString method to create a new + // std::unique_ptr object. + // + // @param offset The offset for the Customizable from the base pointer + // @param ovt How to verify this option + // @param flags, Extra flags specifying the behavior of this option + // @param _sfunc Optional function for serializing this option + // @param _efunc Optional function for comparing this option + template + static OptionTypeInfo AsCustomUniquePtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags) { + return AsCustomUniquePtr(offset, ovt, flags, nullptr, nullptr); + } + + template + static OptionTypeInfo AsCustomUniquePtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags, + const SerializeFunc& serialize_func, + const EqualsFunc& equals_func) { + return OptionTypeInfo( + offset, OptionType::kCustomizable, ovt, + flags | OptionTypeFlags::kUnique, + [](const ConfigOptions& opts, const std::string&, + const std::string& value, char* addr) { + auto* unique = reinterpret_cast*>(addr); + return T::CreateFromString(opts, value, unique); + }, + serialize_func, equals_func); + } + + // Create a new Customizable* OptionTypeInfo + // This function will call the T::CreateFromString method to create a new + // T object. + // + // @param _offset The offset for the Customizable from the base pointer + // @param ovt How to verify this option + // @param flags, Extra flags specifying the behavior of this option + // @param _sfunc Optional function for serializing this option + // @param _efunc Optional function for comparing this option + template + static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt, + OptionTypeFlags flags) { + return AsCustomRawPtr(offset, ovt, flags, nullptr, nullptr); + } + + template + static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt, + OptionTypeFlags flags, + const SerializeFunc& serialize_func, + const EqualsFunc& equals_func) { + return OptionTypeInfo( + offset, OptionType::kCustomizable, ovt, + flags | OptionTypeFlags::kRawPointer, + [](const ConfigOptions& opts, const std::string&, + const std::string& value, char* addr) { + auto** pointer = reinterpret_cast(addr); + return T::CreateFromString(opts, value, pointer); + }, + serialize_func, equals_func); + } + bool IsEnabled(OptionTypeFlags otf) const { return (flags_ & otf) == otf; } bool IsMutable() const { return IsEnabled(OptionTypeFlags::kMutable); } @@ -475,7 +574,12 @@ class OptionTypeInfo { bool IsStruct() const { return (type_ == OptionType::kStruct); } - bool IsConfigurable() const { return (type_ == OptionType::kConfigurable); } + bool IsConfigurable() const { + return (type_ == OptionType::kConfigurable || + type_ == OptionType::kCustomizable); + } + + bool IsCustomizable() const { return (type_ == OptionType::kCustomizable); } // Returns the underlying pointer for the type at base_addr // The value returned is the underlying "raw" pointer, offset from base. @@ -660,6 +764,10 @@ Status ParseVector(const ConfigOptions& config_options, result->clear(); Status status; + // Turn off ignore_unknown_objects so we can tell if the returned + // object is valid or not. + ConfigOptions copy = config_options; + copy.ignore_unsupported_options = false; for (size_t start = 0, end = 0; status.ok() && start < value.size() && end != std::string::npos; start = end + 1) { @@ -667,10 +775,15 @@ Status ParseVector(const ConfigOptions& config_options, status = OptionTypeInfo::NextToken(value, separator, start, &end, &token); if (status.ok()) { T elem; - status = elem_info.Parse(config_options, name, token, - reinterpret_cast(&elem)); + status = + elem_info.Parse(copy, name, token, reinterpret_cast(&elem)); if (status.ok()) { result->emplace_back(elem); + } else if (config_options.ignore_unsupported_options && + status.IsNotSupported()) { + // If we were ignoring unsupported options and this one should be + // ignored, ignore it by setting the status to OK + status = Status::OK(); } } } diff --git a/options/cf_options.cc b/options/cf_options.cc index caa4aa29c..5ddd8fa81 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -567,31 +567,15 @@ static std::unordered_map } return s; }}}, - {"table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kConfigurable, OptionVerificationType::kByName, - (OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose | - OptionTypeFlags::kDontPrepare), - // Creates a new TableFactory based on value - [](const ConfigOptions& opts, const std::string& /*name*/, - const std::string& value, char* addr) { - auto table_factory = - reinterpret_cast*>(addr); - return TableFactory::CreateFromString(opts, value, table_factory); - }, - // Converts the TableFactory into its string representation - [](const ConfigOptions& /*opts*/, const std::string& /*name*/, - const char* addr, std::string* value) { - const auto* table_factory = - reinterpret_cast*>(addr); - *value = table_factory->get() ? table_factory->get()->Name() - : kNullptrString; - return Status::OK(); - }, - /* No equals function for table factories */ nullptr}}, + {"table_factory", OptionTypeInfo::AsCustomSharedPtr( + offset_of(&ColumnFamilyOptions::table_factory), + OptionVerificationType::kByName, + (OptionTypeFlags::kCompareLoose | + OptionTypeFlags::kStringNameOnly | + OptionTypeFlags::kDontPrepare))}, {"block_based_table_factory", {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kConfigurable, OptionVerificationType::kAlias, + OptionType::kCustomizable, OptionVerificationType::kAlias, OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, // Parses the input value and creates a BlockBasedTableFactory [](const ConfigOptions& opts, const std::string& name, @@ -623,7 +607,7 @@ static std::unordered_map }}}, {"plain_table_factory", {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kConfigurable, OptionVerificationType::kAlias, + OptionType::kCustomizable, OptionVerificationType::kAlias, OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, // Parses the input value and creates a PlainTableFactory [](const ConfigOptions& opts, const std::string& name, diff --git a/options/configurable.cc b/options/configurable.cc index 8c11b0b0e..3167c5d06 100644 --- a/options/configurable.cc +++ b/options/configurable.cc @@ -8,6 +8,7 @@ #include "logging/logging.h" #include "options/configurable_helper.h" #include "options/options_helper.h" +#include "rocksdb/customizable.h" #include "rocksdb/status.h" #include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/options_type.h" @@ -57,13 +58,9 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) { } } } +#else + (void)opts; #endif // ROCKSDB_LITE - if (status.ok()) { - auto inner = Inner(); - if (inner != nullptr) { - status = inner->PrepareOptions(opts); - } - } if (status.ok()) { prepared_ = true; } @@ -94,13 +91,10 @@ Status Configurable::ValidateOptions(const DBOptions& db_opts, } } } +#else + (void)db_opts; + (void)cf_opts; #endif // ROCKSDB_LITE - if (status.ok()) { - const auto inner = Inner(); - if (inner != nullptr) { - status = inner->ValidateOptions(db_opts, cf_opts); - } - } return status; } @@ -116,12 +110,7 @@ const void* Configurable::GetOptionsPtr(const std::string& name) const { return o.opt_ptr; } } - auto inner = Inner(); - if (inner != nullptr) { - return inner->GetOptionsPtr(name); - } else { - return nullptr; - } + return nullptr; } std::string Configurable::GetOptionName(const std::string& opt_name) const { @@ -394,6 +383,23 @@ Status ConfigurableHelper::ConfigureOption( if (opt_name == name) { return configurable.ParseOption(config_options, opt_info, opt_name, value, opt_ptr); + } else if (opt_info.IsCustomizable() && + EndsWith(opt_name, ConfigurableHelper::kIdPropSuffix)) { + return configurable.ParseOption(config_options, opt_info, name, value, + opt_ptr); + + } else if (opt_info.IsCustomizable()) { + Customizable* custom = opt_info.AsRawPointer(opt_ptr); + if (value.empty()) { + return Status::OK(); + } else if (custom == nullptr || !StartsWith(name, custom->GetId() + ".")) { + return configurable.ParseOption(config_options, opt_info, name, value, + opt_ptr); + } else if (value.find("=") != std::string::npos) { + return custom->ConfigureFromString(config_options, value); + } else { + return custom->ConfigureOption(config_options, name, value); + } } else if (opt_info.IsStruct() || opt_info.IsConfigurable()) { return configurable.ParseOption(config_options, opt_info, name, value, opt_ptr); @@ -403,6 +409,32 @@ Status ConfigurableHelper::ConfigureOption( } #endif // ROCKSDB_LITE +Status ConfigurableHelper::ConfigureNewObject( + const ConfigOptions& config_options_in, Configurable* object, + const std::string& id, const std::string& base_opts, + const std::unordered_map& opts) { + if (object != nullptr) { + ConfigOptions config_options = config_options_in; + config_options.invoke_prepare_options = false; + if (!base_opts.empty()) { +#ifndef ROCKSDB_LITE + // Don't run prepare options on the base, as we would do that on the + // overlay opts instead + Status status = object->ConfigureFromString(config_options, base_opts); + if (!status.ok()) { + return status; + } +#endif // ROCKSDB_LITE + } + if (!opts.empty()) { + return object->ConfigureFromMap(config_options, opts); + } + } else if (!opts.empty()) { // No object but no map. This is OK + return Status::InvalidArgument("Cannot configure null object ", id); + } + return Status::OK(); +} + //******************************************************************************* // // Methods for Converting Options into strings @@ -607,4 +639,43 @@ bool ConfigurableHelper::AreEquivalent(const ConfigOptions& config_options, return true; } #endif // ROCKSDB_LITE + +Status ConfigurableHelper::GetOptionsMap( + const std::string& value, std::string* id, + std::unordered_map* props) { + return GetOptionsMap(value, "", id, props); +} + +Status ConfigurableHelper::GetOptionsMap( + const std::string& value, const std::string& default_id, std::string* id, + std::unordered_map* props) { + assert(id); + assert(props); + Status status; + if (value.empty() || value == kNullptrString) { + *id = default_id; + } else if (value.find('=') == std::string::npos) { + *id = value; +#ifndef ROCKSDB_LITE + } else { + status = StringToMap(value, props); + if (status.ok()) { + auto iter = props->find(ConfigurableHelper::kIdPropName); + if (iter != props->end()) { + *id = iter->second; + props->erase(iter); + } else if (default_id.empty()) { // Should this be an error?? + status = Status::InvalidArgument("Name property is missing"); + } else { + *id = default_id; + } + } +#else + } else { + *id = value; + props->clear(); +#endif + } + return status; +} } // namespace ROCKSDB_NAMESPACE diff --git a/options/configurable_helper.h b/options/configurable_helper.h index 6a2454727..adc070548 100644 --- a/options/configurable_helper.h +++ b/options/configurable_helper.h @@ -20,6 +20,8 @@ namespace ROCKSDB_NAMESPACE { // of configuring the objects. class ConfigurableHelper { public: + constexpr static const char* kIdPropName = "id"; + constexpr static const char* kIdPropSuffix = ".id"; // Registers the input name with the options and associated map. // When classes register their options in this manner, most of the // functionality (excluding unknown options and validate/prepare) is @@ -75,6 +77,43 @@ class ConfigurableHelper { const std::unordered_map& options, std::unordered_map* unused); + // Helper method for configuring a new customizable object. + // If base_opts are set, this is the "default" options to use for the new + // object. Then any values in "new_opts" are applied to the object. + // Returns OK if the object could be successfully configured + // @return NotFound If any of the names in the base or new opts were not valid + // for this object. + // @return NotSupported If any of the names are valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If any of the values cannot be successfully + // parsed. + static Status ConfigureNewObject( + const ConfigOptions& config_options, Configurable* object, + const std::string& id, const std::string& base_opts, + const std::unordered_map& new_opts); + + // Splits the input opt_value into the ID field and the remaining options. + // The input opt_value can be in the form of "name" or "name=value + // [;name=value]". The first form uses the "name" as an id with no options The + // latter form converts the input into a map of name=value pairs and sets "id" + // to the "id" value from the map. + // @param opt_value The value to split into id and options + // @param id The id field from the opt_value + // @param options The remaining name/value pairs from the opt_value + // @param default_id If specified and there is no id field in the map, this + // value is returned as the ID + // @return OK if the value was converted to a map succesfully and an ID was + // found. + // @return InvalidArgument if the value could not be converted to a map or + // there was or there is no id property in the map. + static Status GetOptionsMap( + const std::string& opt_value, std::string* id, + std::unordered_map* options); + static Status GetOptionsMap( + const std::string& opt_value, const std::string& default_id, + std::string* id, std::unordered_map* options); + #ifndef ROCKSDB_LITE // Internal method to configure a set of options for this object. // Classes may override this value to change its behavior. @@ -205,6 +244,7 @@ class ConfigurableHelper { static const OptionTypeInfo* FindOption( const std::vector& options, const std::string& name, std::string* opt_name, void** opt_ptr); + #endif // ROCKSDB_LITE }; diff --git a/options/configurable_test.cc b/options/configurable_test.cc index 27f877526..fe0c76561 100644 --- a/options/configurable_test.cc +++ b/options/configurable_test.cc @@ -79,29 +79,6 @@ class SimpleConfigurable : public TestConfigurable { }; // End class SimpleConfigurable -static std::unordered_map wrapped_option_info = { -#ifndef ROCKSDB_LITE - {"inner", - {0, OptionType::kConfigurable, OptionVerificationType::kNormal, - OptionTypeFlags::kShared}}, -#endif // ROCKSDB_LITE -}; -class WrappedConfigurable : public SimpleConfigurable { - public: - WrappedConfigurable(const std::string& name, unsigned char mode, - const std::shared_ptr& t) - : SimpleConfigurable(name, mode, &simple_option_info), inner_(t) { - ConfigurableHelper::RegisterOptions(*this, "WrappedOptions", &inner_, - &wrapped_option_info); - } - - protected: - Configurable* Inner() const override { return inner_.get(); } - - private: - std::shared_ptr inner_; -}; - using ConfigTestFactoryFunc = std::function; class ConfigurableTest : public testing::Test { @@ -607,17 +584,6 @@ static std::unordered_map TestFactories = { TestConfigMode::kSimpleMode | TestConfigMode::kNestedMode); }}, - {"ThreeWay", - []() { - std::shared_ptr child; - child.reset( - SimpleConfigurable::Create("child", TestConfigMode::kDefaultMode)); - std::shared_ptr parent; - parent.reset(new WrappedConfigurable( - "parent", TestConfigMode::kDefaultMode, child)); - return new WrappedConfigurable("master", TestConfigMode::kDefaultMode, - parent); - }}, {"ThreeDeep", []() { Configurable* simple = SimpleConfigurable::Create( @@ -765,10 +731,6 @@ INSTANTIATE_TEST_CASE_P( "pointer={int=22;string=pointer};" "unique={int=33;string=unique};" "shared={int=44;string=shared}"), - std::pair("ThreeWay", - "int=11;bool=true;string=outer;" - "inner={int=22;string=parent;" - "inner={int=33;string=child}};"), std::pair("ThreeDeep", "int=11;bool=true;string=outer;" "unique={int=22;string=inner;" diff --git a/options/customizable.cc b/options/customizable.cc new file mode 100644 index 000000000..3488f326b --- /dev/null +++ b/options/customizable.cc @@ -0,0 +1,77 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/customizable.h" + +#include "options/configurable_helper.h" +#include "rocksdb/convenience.h" +#include "rocksdb/status.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +std::string Customizable::GetOptionName(const std::string& long_name) const { + const std::string& name = Name(); + size_t name_len = name.size(); + if (long_name.size() > name_len + 1 && + long_name.compare(0, name_len, name) == 0 && + long_name.at(name_len) == '.') { + return long_name.substr(name_len + 1); + } else { + return Configurable::GetOptionName(long_name); + } +} + +#ifndef ROCKSDB_LITE +Status Customizable::GetOption(const ConfigOptions& config_options, + const std::string& opt_name, + std::string* value) const { + if (opt_name == ConfigurableHelper::kIdPropName) { + *value = GetId(); + return Status::OK(); + } else { + return Configurable::GetOption(config_options, opt_name, value); + } +} + +std::string Customizable::SerializeOptions(const ConfigOptions& config_options, + const std::string& prefix) const { + std::string result; + std::string parent; + if (!config_options.IsShallow()) { + parent = Configurable::SerializeOptions(config_options, ""); + } + if (parent.empty()) { + result = GetId(); + } else { + result.append(prefix + ConfigurableHelper::kIdPropName + "=" + GetId() + + config_options.delimiter); + result.append(parent); + } + return result; +} + +#endif // ROCKSDB_LITE + +bool Customizable::AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* mismatch) const { + if (config_options.sanity_level > ConfigOptions::kSanityLevelNone && + this != other) { + const Customizable* custom = reinterpret_cast(other); + if (GetId() != custom->GetId()) { + *mismatch = ConfigurableHelper::kIdPropName; + return false; + } else if (config_options.sanity_level > + ConfigOptions::kSanityLevelLooselyCompatible) { + bool matches = + Configurable::AreEquivalent(config_options, other, mismatch); + return matches; + } + } + return true; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/options/customizable_helper.h b/options/customizable_helper.h new file mode 100644 index 000000000..c9f6f747e --- /dev/null +++ b/options/customizable_helper.h @@ -0,0 +1,216 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include +#include + +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "rocksdb/convenience.h" +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" +#include "rocksdb/utilities/object_registry.h" + +namespace ROCKSDB_NAMESPACE { +template +using SharedFactoryFunc = + std::function*)>; + +template +using UniqueFactoryFunc = + std::function*)>; + +template +using StaticFactoryFunc = std::function; + +// Creates a new shared Customizable object based on the input parameters. +// This method parses the input value to determine the type of instance to +// create. If there is an existing instance (in result) and it is the same type +// as the object being created, the existing configuration is stored and used as +// the default for the new object. +// +// The value parameter specified the instance class of the object to create. +// If it is a simple string (e.g. BlockBasedTable), then the instance will be +// created using the default settings. If the value is a set of name-value +// pairs, then the "id" value is used to determine the instance to create and +// the remaining parameters are used to configure the object. Id name-value +// pairs are specified, there must be an "id=value" pairing or an error will +// result. +// +// The config_options parameter controls the process and how errors are +// returned. If ignore_unknown_options=true, unknown values are ignored during +// the configuration If ignore_unsupported_options=true, unknown instance types +// are ignored If invoke_prepare_options=true, the resulting instance wll be +// initialized (via PrepareOptions +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param value Either the simple name of the instance to create, or a set of +// name-value pairs to +// create and initailzie the object +// @param func Optional function to call to attempt to create an instance +// @param result The newly created instance. +template +static Status LoadSharedObject(const ConfigOptions& config_options, + const std::string& value, + const SharedFactoryFunc& func, + std::shared_ptr* result) { + std::string id; + std::unordered_map opt_map; + Status status = ConfigurableHelper::GetOptionsMap(value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } + std::string curr_opts; +#ifndef ROCKSDB_LITE + if (result->get() != nullptr && result->get()->GetId() == id) { + // Try to get the existing options, ignoring any errors + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + result->get()->GetOptionString(embedded, &curr_opts).PermitUncheckedError(); + } +#endif + if (func == nullptr || !func(id, result)) { // No factory, or it failed + if (id.empty() && opt_map.empty()) { + // No Id and no options. Clear the object + result->reset(); + return Status::OK(); + } else if (id.empty()) { // We have no Id but have options. Not good + return Status::NotSupported("Cannot reset object ", id); + } else { +#ifndef ROCKSDB_LITE + status = ObjectRegistry::NewInstance()->NewSharedObject(id, result); +#else + status = Status::NotSupported("Cannot load object in LITE mode ", id); +#endif + if (!status.ok()) { + if (config_options.ignore_unsupported_options) { + return Status::OK(); + } else { + return status; + } + } + } + } + return ConfigurableHelper::ConfigureNewObject(config_options, result->get(), + id, curr_opts, opt_map); +} + +// Creates a new unique customizable instance object based on the input +// parameters. +// @see LoadSharedObject for more information on the inner workings of this +// method. +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param value Either the simple name of the instance to create, or a set of +// name-value pairs to +// create and initailzie the object +// @param func Optional function to call to attempt to create an instance +// @param result The newly created instance. +template +static Status LoadUniqueObject(const ConfigOptions& config_options, + const std::string& value, + const UniqueFactoryFunc& func, + std::unique_ptr* result) { + std::string id; + std::unordered_map opt_map; + Status status = ConfigurableHelper::GetOptionsMap(value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } + std::string curr_opts; +#ifndef ROCKSDB_LITE + if (result->get() != nullptr && result->get()->GetId() == id) { + // Try to get the existing options, ignoring any errors + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + result->get()->GetOptionString(embedded, &curr_opts).PermitUncheckedError(); + } +#endif + if (func == nullptr || !func(id, result)) { // No factory, or it failed + if (id.empty() && opt_map.empty()) { + // No Id and no options. Clear the object + result->reset(); + return Status::OK(); + } else if (id.empty()) { // We have no Id but have options. Not good + return Status::NotSupported("Cannot reset object ", id); + } else { +#ifndef ROCKSDB_LITE + status = ObjectRegistry::NewInstance()->NewUniqueObject(id, result); +#else + status = Status::NotSupported("Cannot load object in LITE mode ", id); +#endif // ROCKSDB_LITE + if (!status.ok()) { + if (config_options.ignore_unsupported_options) { + return Status::OK(); + } else { + return status; + } + } + } + } + return ConfigurableHelper::ConfigureNewObject(config_options, result->get(), + id, curr_opts, opt_map); +} +// Creates a new static (raw pointer) customizable instance object based on the +// input parameters. +// @see LoadSharedObject for more information on the inner workings of this +// method. +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param value Either the simple name of the instance to create, or a set of +// name-value pairs to +// create and initailzie the object +// @param func Optional function to call to attempt to create an instance +// @param result The newly created instance. +template +static Status LoadStaticObject(const ConfigOptions& config_options, + const std::string& value, + const StaticFactoryFunc& func, T** result) { + std::string id; + std::unordered_map opt_map; + Status status = ConfigurableHelper::GetOptionsMap(value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } + std::string curr_opts; +#ifndef ROCKSDB_LITE + if (*result != nullptr && (*result)->GetId() == id) { + // Try to get the existing options, ignoring any errors + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + (*result)->GetOptionString(embedded, &curr_opts).PermitUncheckedError(); + } +#endif + if (func == nullptr || !func(id, result)) { // No factory, or it failed + if (id.empty() && opt_map.empty()) { + // No Id and no options. Clear the object + *result = nullptr; + return Status::OK(); + } else if (id.empty()) { // We have no Id but have options. Not good + return Status::NotSupported("Cannot reset object ", id); + } else { +#ifndef ROCKSDB_LITE + status = ObjectRegistry::NewInstance()->NewStaticObject(id, result); +#else + status = Status::NotSupported("Cannot load object in LITE mode ", id); +#endif // ROCKSDB_LITE + if (!status.ok()) { + if (config_options.ignore_unsupported_options) { + return Status::OK(); + } else { + return status; + } + } + } + } + return ConfigurableHelper::ConfigureNewObject(config_options, *result, id, + curr_opts, opt_map); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/options/customizable_test.cc b/options/customizable_test.cc new file mode 100644 index 000000000..100ed6787 --- /dev/null +++ b/options/customizable_test.cc @@ -0,0 +1,625 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/customizable.h" + +#include +#include +#include +#include + +#include "options/configurable_helper.h" +#include "options/customizable_helper.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "rocksdb/convenience.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "table/mock_table.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace ROCKSDB_NAMESPACE { +class StringLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char buffer[1000]; + vsnprintf(buffer, sizeof(buffer), format, ap); + string_.append(buffer); + } + const std::string& str() const { return string_; } + void clear() { string_.clear(); } + + private: + std::string string_; +}; + +class TestCustomizable : public Customizable { + public: + TestCustomizable(const std::string& name) : name_(name) {} + // Method to allow CheckedCast to work for this class + static const char* kClassName() { + return "TestCustomizable"; + ; + } + + const char* Name() const override { return name_.c_str(); } + static const char* Type() { return "test.custom"; } + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + std::unique_ptr* result); + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + std::shared_ptr* result); + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + TestCustomizable** result); + bool IsInstanceOf(const std::string& name) const override { + if (name == kClassName()) { + return true; + } else { + return Customizable::IsInstanceOf(name); + } + } + + protected: + const std::string name_; +}; + +struct AOptions { + int i = 0; + bool b = false; +}; + +static std::unordered_map a_option_info = { +#ifndef ROCKSDB_LITE + {"int", + {offsetof(struct AOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bool", + {offsetof(struct AOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; +class ACustomizable : public TestCustomizable { + public: + ACustomizable(const std::string& id) : TestCustomizable("A"), id_(id) { + ConfigurableHelper::RegisterOptions(*this, "A", &opts_, &a_option_info); + } + std::string GetId() const override { return id_; } + static const char* kClassName() { return "A"; } + + private: + AOptions opts_; + const std::string id_; +}; + +#ifndef ROCKSDB_LITE +static int A_count = 0; +const FactoryFunc& a_func = + ObjectLibrary::Default()->Register( + "A.*", + [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new ACustomizable(name)); + A_count++; + return guard->get(); + }); +#endif // ROCKSDB_LITE + +struct BOptions { + std::string s; + bool b = false; +}; + +static std::unordered_map b_option_info = { +#ifndef ROCKSDB_LITE + {"string", + {offsetof(struct BOptions, s), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bool", + {offsetof(struct BOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; + +class BCustomizable : public TestCustomizable { + private: + public: + BCustomizable(const std::string& name) : TestCustomizable(name) { + ConfigurableHelper::RegisterOptions(*this, name, &opts_, &b_option_info); + } + static const char* kClassName() { return "B"; } + + private: + BOptions opts_; +}; + +static bool LoadSharedB(const std::string& id, + std::shared_ptr* result) { + if (id == "B") { + result->reset(new BCustomizable(id)); + return true; + } else if (id.empty()) { + result->reset(); + return true; + } else { + return false; + } +} +Status TestCustomizable::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + return LoadSharedObject(config_options, value, LoadSharedB, + result); +} + +Status TestCustomizable::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::unique_ptr* result) { + return LoadUniqueObject( + config_options, value, + [](const std::string& id, std::unique_ptr* u) { + if (id == "B") { + u->reset(new BCustomizable(id)); + return true; + } else if (id.empty()) { + u->reset(); + return true; + } else { + return false; + } + }, + result); +} + +Status TestCustomizable::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + TestCustomizable** result) { + return LoadStaticObject( + config_options, value, + [](const std::string& id, TestCustomizable** ptr) { + if (id == "B") { + *ptr = new BCustomizable(id); + return true; + } else if (id.empty()) { + *ptr = nullptr; + return true; + } else { + return false; + } + }, + result); +} + +#ifndef ROCKSDB_LITE +const FactoryFunc& s_func = + ObjectLibrary::Default()->Register( + "S", [](const std::string& name, + std::unique_ptr* /* guard */, + std::string* /* msg */) { return new BCustomizable(name); }); +#endif // ROCKSDB_LITE + +struct SimpleOptions { + bool b = true; + std::unique_ptr cu; + std::shared_ptr cs; + TestCustomizable* cp = nullptr; +}; + +static std::unordered_map simple_option_info = { +#ifndef ROCKSDB_LITE + {"bool", + {offsetof(struct SimpleOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"unique", OptionTypeInfo::AsCustomUniquePtr( + offsetof(struct SimpleOptions, cu), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"shared", OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct SimpleOptions, cs), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"pointer", OptionTypeInfo::AsCustomRawPtr( + offsetof(struct SimpleOptions, cp), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, +#endif // ROCKSDB_LITE +}; + +class SimpleConfigurable : public Configurable { + private: + SimpleOptions simple_; + + public: + SimpleConfigurable() { + ConfigurableHelper::RegisterOptions(*this, "simple", &simple_, + &simple_option_info); + } + + SimpleConfigurable( + const std::unordered_map* map) { + ConfigurableHelper::RegisterOptions(*this, "simple", &simple_, map); + } +}; + +class CustomizableTest : public testing::Test { + public: + ConfigOptions config_options_; +}; + +#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE +// Tests that a Customizable can be created by: +// - a simple name +// - a XXX.id option +// - a property with a name +TEST_F(CustomizableTest, CreateByNameTest) { + ObjectLibrary::Default()->Register( + "TEST.*", + [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new TestCustomizable(name)); + return guard->get(); + }); + std::unique_ptr configurable(new SimpleConfigurable()); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique={id=TEST_1}")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_1"); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique.id=TEST_2")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_2"); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique=TEST_3")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_3"); +} + +TEST_F(CustomizableTest, ToStringTest) { + std::unique_ptr custom(new TestCustomizable("test")); + ASSERT_EQ(custom->ToString(config_options_), "test"); +} + +TEST_F(CustomizableTest, SimpleConfigureTest) { + std::unordered_map opt_map = { + {"unique", "id=A;int=1;bool=true"}, + {"shared", "id=B;string=s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); + std::string opt_str; + std::string mismatch; + ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str)); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE( + configurable->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +static void GetMapFromProperties( + const std::string& props, + std::unordered_map* map) { + std::istringstream iss(props); + std::unordered_map copy_map; + std::string line; + map->clear(); + for (int line_num = 0; std::getline(iss, line); line_num++) { + std::string name; + std::string value; + ASSERT_OK( + RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num)); + (*map)[name] = value; + } +} + +TEST_F(CustomizableTest, ConfigureFromPropsTest) { + std::unordered_map opt_map = { + {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"}, + {"shared.id", "B"}, {"shared.B.string", "s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); + std::string opt_str; + std::string mismatch; + config_options_.delimiter = "\n"; + std::unordered_map props; + ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str)); + GetMapFromProperties(opt_str, &props); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromMap(config_options_, props)); + ASSERT_TRUE( + configurable->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CustomizableTest, ConfigureFromShortTest) { + std::unordered_map opt_map = { + {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"}, + {"shared.id", "B"}, {"shared.B.string", "s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); +} + +TEST_F(CustomizableTest, AreEquivalentOptionsTest) { + std::unordered_map opt_map = { + {"unique", "id=A;int=1;bool=true"}, + {"shared", "id=A;int=1;bool=true"}, + }; + std::string mismatch; + ConfigOptions config_options = config_options_; + config_options.invoke_prepare_options = false; + std::unique_ptr c1(new SimpleConfigurable()); + std::unique_ptr c2(new SimpleConfigurable()); + ASSERT_OK(c1->ConfigureFromMap(config_options, opt_map)); + ASSERT_OK(c2->ConfigureFromMap(config_options, opt_map)); + ASSERT_TRUE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + SimpleOptions* simple = c1->GetOptions("simple"); + ASSERT_TRUE( + simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch)); + ASSERT_OK(simple->cu->ConfigureOption(config_options, "int", "2")); + ASSERT_FALSE( + simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + ConfigOptions loosely = config_options; + loosely.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_TRUE(simple->cu->AreEquivalent(loosely, simple->cs.get(), &mismatch)); + + ASSERT_OK(c1->ConfigureOption(config_options, "shared", "id=B;string=3")); + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + ASSERT_FALSE(simple->cs->AreEquivalent(loosely, simple->cu.get(), &mismatch)); + simple->cs.reset(); + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); +} + +// Tests that we can initialize a customizable from its options +TEST_F(CustomizableTest, ConfigureStandaloneCustomTest) { + std::unique_ptr base, copy; + auto registry = ObjectRegistry::NewInstance(); + ASSERT_OK(registry->NewUniqueObject("A", &base)); + ASSERT_OK(registry->NewUniqueObject("A", ©)); + ASSERT_OK(base->ConfigureFromString(config_options_, "int=33;bool=true")); + std::string opt_str; + std::string mismatch; + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +// Tests that we fail appropriately if the pattern is not registered +TEST_F(CustomizableTest, BadNameTest) { + config_options_.ignore_unsupported_options = false; + std::unique_ptr c1(new SimpleConfigurable()); + ASSERT_NOK( + c1->ConfigureFromString(config_options_, "unique.shared.id=bad name")); + config_options_.ignore_unsupported_options = true; + ASSERT_OK( + c1->ConfigureFromString(config_options_, "unique.shared.id=bad name")); +} + +// Tests that we fail appropriately if a bad option is passed to the underlying +// configurable +TEST_F(CustomizableTest, BadOptionTest) { + std::unique_ptr c1(new SimpleConfigurable()); + ConfigOptions ignore = config_options_; + ignore.ignore_unknown_options = true; + + ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.int=11")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "shared={id=B;int=1}")); + ASSERT_OK(c1->ConfigureFromString(ignore, "shared={id=A;string=s}")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "B.int=11")); + ASSERT_OK(c1->ConfigureFromString(ignore, "B.int=11")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.string=s")); + ASSERT_OK(c1->ConfigureFromString(ignore, "A.string=s")); + // Test as detached + ASSERT_NOK( + c1->ConfigureFromString(config_options_, "shared.id=A;A.string=b}")); + ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=A;A.string=s}")); +} + +// Tests that different IDs lead to different objects +TEST_F(CustomizableTest, UniqueIdTest) { + std::unique_ptr base(new SimpleConfigurable()); + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=true}")); + SimpleOptions* simple = base->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), std::string("A_1")); + std::string opt_str; + std::string mismatch; + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_2;int=1;bool=true}")); + ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_EQ(simple->cu->GetId(), std::string("A_2")); +} + +TEST_F(CustomizableTest, IsInstanceOfTest) { + std::shared_ptr tc = std::make_shared("A"); + + ASSERT_TRUE(tc->IsInstanceOf("A")); + ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable")); + ASSERT_FALSE(tc->IsInstanceOf("B")); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), nullptr); + + tc.reset(new BCustomizable("B")); + ASSERT_TRUE(tc->IsInstanceOf("B")); + ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable")); + ASSERT_FALSE(tc->IsInstanceOf("A")); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), nullptr); +} + +static std::unordered_map inner_option_info = { +#ifndef ROCKSDB_LITE + {"inner", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kStringNameOnly)} +#endif // ROCKSDB_LITE +}; + +class ShallowCustomizable : public Customizable { + public: + ShallowCustomizable() { + inner_ = std::make_shared("a"); + ConfigurableHelper::RegisterOptions(*this, "inner", &inner_, + &inner_option_info); + }; + static const char* kClassName() { return "shallow"; } + const char* Name() const override { return kClassName(); } + + private: + std::shared_ptr inner_; +}; + +TEST_F(CustomizableTest, TestStringDepth) { + ConfigOptions shallow = config_options_; + std::unique_ptr c(new ShallowCustomizable()); + std::string opt_str; + shallow.depth = ConfigOptions::Depth::kDepthShallow; + ASSERT_OK(c->GetOptionString(shallow, &opt_str)); + ASSERT_EQ(opt_str, "inner=a;"); + shallow.depth = ConfigOptions::Depth::kDepthDetailed; + ASSERT_OK(c->GetOptionString(shallow, &opt_str)); + ASSERT_NE(opt_str, "inner=a;"); +} + +// Tests that we only get a new customizable when it changes +TEST_F(CustomizableTest, NewCustomizableTest) { + std::unique_ptr base(new SimpleConfigurable()); + A_count = 0; + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=true}")); + SimpleOptions* simple = base->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(A_count, 1); // Created one A + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=false}")); + ASSERT_EQ(A_count, 2); // Create another A_1 + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_2;int=1;bool=false}")); + ASSERT_EQ(A_count, 3); // Created another A + ASSERT_OK(base->ConfigureFromString(config_options_, "unique=")); + ASSERT_EQ(simple->cu, nullptr); + ASSERT_EQ(A_count, 3); +} + +TEST_F(CustomizableTest, IgnoreUnknownObjects) { + ConfigOptions ignore = config_options_; + std::shared_ptr shared; + std::unique_ptr unique; + TestCustomizable* pointer = nullptr; + ignore.ignore_unsupported_options = false; + ASSERT_NOK( + LoadSharedObject(ignore, "Unknown", nullptr, &shared)); + ASSERT_NOK( + LoadUniqueObject(ignore, "Unknown", nullptr, &unique)); + ASSERT_NOK( + LoadStaticObject(ignore, "Unknown", nullptr, &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ignore.ignore_unsupported_options = true; + ASSERT_OK( + LoadSharedObject(ignore, "Unknown", nullptr, &shared)); + ASSERT_OK( + LoadUniqueObject(ignore, "Unknown", nullptr, &unique)); + ASSERT_OK( + LoadStaticObject(ignore, "Unknown", nullptr, &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_OK(LoadSharedObject(ignore, "id=Unknown", nullptr, + &shared)); + ASSERT_OK(LoadUniqueObject(ignore, "id=Unknown", nullptr, + &unique)); + ASSERT_OK(LoadStaticObject(ignore, "id=Unknown", nullptr, + &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_OK(LoadSharedObject(ignore, "id=Unknown;option=bad", + nullptr, &shared)); + ASSERT_OK(LoadUniqueObject(ignore, "id=Unknown;option=bad", + nullptr, &unique)); + ASSERT_OK(LoadStaticObject(ignore, "id=Unknown;option=bad", + nullptr, &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); +} + +TEST_F(CustomizableTest, FactoryFunctionTest) { + std::shared_ptr shared; + std::unique_ptr unique; + TestCustomizable* pointer = nullptr; + ConfigOptions ignore = config_options_; + ignore.ignore_unsupported_options = false; + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &shared)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &unique)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &pointer)); + ASSERT_NE(shared.get(), nullptr); + ASSERT_NE(unique.get(), nullptr); + ASSERT_NE(pointer, nullptr); + delete pointer; + pointer = nullptr; + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &shared)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &unique)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &shared)); + ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &unique)); + ASSERT_NOK( + TestCustomizable::CreateFromString(ignore, "option=bad", &pointer)); + ASSERT_EQ(pointer, nullptr); +} + +#endif // !ROCKSDB_LITE + +} // namespace ROCKSDB_NAMESPACE +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/options/options_helper.cc b/options/options_helper.cc index f1ed4eda3..a0a3ba1d6 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -1062,6 +1062,19 @@ Status OptionTypeInfo::Serialize(const ConfigOptions& config_options, return serialize_func_(config_options, opt_name, opt_addr, opt_value); } else if (SerializeSingleOptionHelper(opt_addr, type_, opt_value)) { return Status::OK(); + } else if (IsCustomizable()) { + const Customizable* custom = AsRawPointer(opt_ptr); + if (custom == nullptr) { + *opt_value = kNullptrString; + } else if (IsEnabled(OptionTypeFlags::kStringNameOnly) && + !config_options.IsDetailed()) { + *opt_value = custom->GetId(); + } else { + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + *opt_value = custom->ToString(embedded); + } + return Status::OK(); } else if (IsConfigurable()) { const Configurable* config = AsRawPointer(opt_ptr); if (config != nullptr) { diff --git a/src.mk b/src.mk index 58e8e841b..ab178f2ef 100644 --- a/src.mk +++ b/src.mk @@ -127,6 +127,7 @@ LIB_SOURCES = \ monitoring/thread_status_util_debug.cc \ options/cf_options.cc \ options/configurable.cc \ + options/customizable.cc \ options/db_options.cc \ options/options.cc \ options/options_helper.cc \ @@ -458,6 +459,7 @@ TEST_MAIN_SOURCES = \ monitoring/statistics_test.cc \ monitoring/stats_history_test.cc \ options/configurable_test.cc \ + options/customizable_test.cc \ options/options_settable_test.cc \ options/options_test.cc \ table/block_based/block_based_filter_block_test.cc \ diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h index a7120f854..a1a95de82 100644 --- a/table/block_based/block_based_table_factory.h +++ b/table/block_based/block_based_table_factory.h @@ -46,6 +46,9 @@ class BlockBasedTableFactory : public TableFactory { ~BlockBasedTableFactory() {} + // Method to allow CheckedCast to work for this class + static const char* kClassName() { return kBlockBasedTableName(); } + const char* Name() const override { return kBlockBasedTableName(); } using TableFactory::NewTableReader; diff --git a/table/cuckoo/cuckoo_table_factory.h b/table/cuckoo/cuckoo_table_factory.h index 30d4155e1..429ad23d2 100644 --- a/table/cuckoo/cuckoo_table_factory.h +++ b/table/cuckoo/cuckoo_table_factory.h @@ -56,6 +56,8 @@ class CuckooTableFactory : public TableFactory { const CuckooTableOptions& table_option = CuckooTableOptions()); ~CuckooTableFactory() {} + // Method to allow CheckedCast to work for this class + static const char* kClassName() { return kCuckooTableName(); } const char* Name() const override { return kCuckooTableName(); } using TableFactory::NewTableReader; diff --git a/table/plain/plain_table_factory.h b/table/plain/plain_table_factory.h index 61a1ed935..d7c8174a7 100644 --- a/table/plain/plain_table_factory.h +++ b/table/plain/plain_table_factory.h @@ -156,6 +156,8 @@ class PlainTableFactory : public TableFactory { explicit PlainTableFactory( const PlainTableOptions& _table_options = PlainTableOptions()); + // Method to allow CheckedCast to work for this class + static const char* kClassName() { return kPlainTableName(); } const char* Name() const override { return kPlainTableName(); } using TableFactory::NewTableReader; Status NewTableReader(const ReadOptions& ro, diff --git a/table/table_factory.cc b/table/table_factory.cc index 5565202e1..962bad9ba 100644 --- a/table/table_factory.cc +++ b/table/table_factory.cc @@ -3,6 +3,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "options/customizable_helper.h" #include "rocksdb/convenience.h" #include "rocksdb/table.h" #include "table/block_based/block_based_table_factory.h" @@ -11,23 +12,9 @@ namespace ROCKSDB_NAMESPACE { -Status TableFactory::CreateFromString(const ConfigOptions& config_options_in, - const std::string& id, - std::shared_ptr* factory) { - Status status; - std::string name = id; - - std::string existing_opts; - - ConfigOptions config_options = config_options_in; - if (factory->get() != nullptr && name == factory->get()->Name()) { - config_options.delimiter = ";"; - - status = factory->get()->GetOptionString(config_options, &existing_opts); - if (!status.ok()) { - return status; - } - } +static bool LoadFactory(const std::string& name, + std::shared_ptr* factory) { + bool success = true; if (name == TableFactory::kBlockBasedTableName()) { factory->reset(new BlockBasedTableFactory()); #ifndef ROCKSDB_LITE @@ -37,14 +24,15 @@ Status TableFactory::CreateFromString(const ConfigOptions& config_options_in, factory->reset(new CuckooTableFactory()); #endif // ROCKSDB_LITE } else { - status = Status::NotSupported("Could not load table factory: ", name); - return status; - } - if (status.ok() && !existing_opts.empty()) { - config_options.invoke_prepare_options = false; - status = factory->get()->ConfigureFromString(config_options, existing_opts); + success = false; } - return status; + return success; } +Status TableFactory::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* factory) { + return LoadSharedObject(config_options, value, LoadFactory, + factory); +} } // namespace ROCKSDB_NAMESPACE