Create a Customizable class to load classes and configurations (#6590)

Summary:
The Customizable class is an extension of the Configurable class and allows instances to be created by a name/ID.  Classes that extend customizable can define their Type (e.g. "TableFactory", "Cache") and  a method to instantiate them (TableFactory::CreateFromString).  Customizable objects can be registered with the ObjectRegistry and created dynamically.

Future PRs will make more types of objects extend Customizable.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/6590

Reviewed By: cheng-chang

Differential Revision: D24841553

Pulled By: zhichao-cao

fbshipit-source-id: d0c2132bd932e971cbfe2c908ca2e5db30c5e155
main
mrambacher 4 years ago committed by Facebook GitHub Bot
parent 8b6b6aeb1a
commit c442f6809f
  1. 2
      CMakeLists.txt
  2. 1
      HISTORY.md
  3. 4
      Makefile
  4. 9
      TARGETS
  5. 5
      include/rocksdb/configurable.h
  6. 138
      include/rocksdb/customizable.h
  7. 20
      include/rocksdb/table.h
  8. 133
      include/rocksdb/utilities/options_type.h
  9. 32
      options/cf_options.cc
  10. 107
      options/configurable.cc
  11. 40
      options/configurable_helper.h
  12. 38
      options/configurable_test.cc
  13. 77
      options/customizable.cc
  14. 216
      options/customizable_helper.h
  15. 625
      options/customizable_test.cc
  16. 13
      options/options_helper.cc
  17. 2
      src.mk
  18. 3
      table/block_based/block_based_table_factory.h
  19. 2
      table/cuckoo/cuckoo_table_factory.h
  20. 2
      table/plain/plain_table_factory.h
  21. 36
      table/table_factory.cc

@ -687,6 +687,7 @@ set(SOURCES
monitoring/thread_status_util_debug.cc
options/cf_options.cc
options/configurable.cc
options/customizable.cc
options/db_options.cc
options/options.cc
options/options_helper.cc
@ -1142,6 +1143,7 @@ if(WITH_TESTS)
monitoring/statistics_test.cc
monitoring/stats_history_test.cc
options/configurable_test.cc
options/customizable_test.cc
options/options_settable_test.cc
options/options_test.cc
table/block_based/block_based_filter_block_test.cc

@ -42,6 +42,7 @@
* Added is_full_compaction to CompactionJobStats, so that the information is available through the EventListener interface.
* Add more stats for MultiGet in Histogram to get number of data blocks, index blocks, filter blocks and sst files read from file system per level.
* SST files have a new table property called db_host_id, which is set to the hostname by default. A new option in DBOptions, db_host_id, allows the property value to be overridden with a user specified string, or disable it completely by making the option string empty.
* Methods to create customizable extensions -- such as TableFactory -- are exposed directly through the Customizable base class (from which these objects inherit). This change will allow these Customizable classes to be loaded and configured in a standard way (via CreateFromString). More information on how to write and use Customizable classes is in the customizable.h header file.
## 6.13 (09/12/2020)
### Bug fixes

@ -628,6 +628,7 @@ ifdef ASSERT_STATUS_CHECKED
plain_table_db_test \
repair_test \
configurable_test \
customizable_test \
options_settable_test \
options_test \
random_test \
@ -1800,6 +1801,9 @@ compact_files_test: $(OBJ_DIR)/db/compact_files_test.o $(TEST_LIBRARY) $(LIBRARY
configurable_test: options/configurable_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
customizable_test: options/customizable_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
options_test: $(OBJ_DIR)/options/options_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)

@ -255,6 +255,7 @@ cpp_library(
"monitoring/thread_status_util_debug.cc",
"options/cf_options.cc",
"options/configurable.cc",
"options/customizable.cc",
"options/db_options.cc",
"options/options.cc",
"options/options_helper.cc",
@ -544,6 +545,7 @@ cpp_library(
"monitoring/thread_status_util_debug.cc",
"options/cf_options.cc",
"options/configurable.cc",
"options/customizable.cc",
"options/db_options.cc",
"options/options.cc",
"options/options_helper.cc",
@ -1090,6 +1092,13 @@ ROCKS_TESTS = [
[],
[],
],
[
"customizable_test",
"options/customizable_test.cc",
"serial",
[],
[],
],
[
"data_block_hash_index_test",
"table/block_based/data_block_hash_index_test.cc",

@ -270,11 +270,6 @@ class Configurable {
// True once the object is prepared. Once the object is prepared, only
// mutable options can be configured.
bool prepared_;
// If this class is a wrapper (has-a), this method should be
// over-written to return the inner configurable (like an EnvWrapper).
// This method should NOT recurse, but should instead return the
// direct Inner object.
virtual Configurable* Inner() const { return nullptr; }
// Returns the raw pointer for the associated named option.
// The name is typically the name of an option registered via the

@ -0,0 +1,138 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include "rocksdb/configurable.h"
#include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE {
/**
* Customizable a base class used by the rocksdb that describes a
* standard way of configuring and creating objects. Customizable objects
* are configurable objects that can be created from an ObjectRegistry.
*
* Customizable classes are used when there are multiple potential
* implementations of a class for use by RocksDB (e.g. Table, Cache,
* MergeOperator, etc). The abstract base class is expected to define a method
* declaring its type and a factory method for creating one of these, such as:
* static const char *Type() { return "Table"; }
* static Status CreateFromString(const ConfigOptions& options,
* const std::string& id,
* std::shared_ptr<TableFactory>* result);
* The "Type" string is expected to be unique (no two base classes are the same
* type). This factory is expected, based on the options and id, create and
* return the appropriate derived type of the customizable class (e.g.
* BlockBasedTableFactory, PlainTableFactory, etc). For extension developers,
* helper classes and methods are provided for writing this factory.
*
* Instances of a Customizable class need to define:
* - A "static const char *kClassName()" method. This method defines the name
* of the class instance (e.g. BlockBasedTable, LRUCache) and is used by the
* CheckedCast method.
* - The Name() of the object. This name is used when creating and saving
* instances of this class. Typically this name will be the same as
* kClassName().
*
* Additionally, Customizable classes should register any options used to
* configure themselves with the Configurable subsystem.
*
* When a Customizable is being created, the "name" property specifies
* the name of the instance being created.
* For custom objects, their configuration and name can be specified by:
* [prop]={name=X;option 1 = value1[; option2=value2...]}
*
* [prop].name=X
* [prop].option1 = value1
*
* [prop].name=X
* X.option1 =value1
*/
class Customizable : public Configurable {
public:
virtual ~Customizable() {}
// Returns the name of this class of Customizable
virtual const char* Name() const = 0;
// Returns an identifier for this Customizable.
// This could be its name or something more complex (like its URL/pattern).
// Used for pretty printing.
virtual std::string GetId() const {
std::string id = Name();
return id;
}
// This is typically determined by if the input name matches the
// name of this object.
// This method is typically used in conjunction with CheckedCast to find the
// derived class instance from its base. For example, if you have an Env
// and want the "Default" env, you would IsInstanceOf("Default") to get
// the default implementation. This method should be used when you need a
// specific derivative or implementation of a class.
//
// Intermediary caches (such as SharedCache) may wish to override this method
// to check for the intermediary name (SharedCache). Classes with multiple
// potential names (e.g. "PosixEnv", "DefaultEnv") may also wish to override
// this method.
//
// @param name The name of the instance to find.
// Returns true if the class is an instance of the input name.
virtual bool IsInstanceOf(const std::string& name) const {
return name == Name();
}
// Returns the named instance of the Customizable as a T*, or nullptr if not
// found. This method uses IsInstanceOf to find the appropriate class instance
// and then casts it to the expected return type.
template <typename T>
const T* CheckedCast() const {
if (IsInstanceOf(T::kClassName())) {
return static_cast<const T*>(this);
} else {
return nullptr;
}
}
template <typename T>
T* CheckedCast() {
if (IsInstanceOf(T::kClassName())) {
return static_cast<T*>(this);
} else {
return nullptr;
}
}
// Checks to see if this Customizable is equivalent to other.
// This method assumes that the two objects are of the same class.
// @param config_options Controls how the options are compared.
// @param other The other object to compare to.
// @param mismatch If the objects do not match, this parameter contains
// the name of the option that triggered the match failure.
// @param True if the objects match, false otherwise.
// @see Configurable::AreEquivalent for more details
bool AreEquivalent(const ConfigOptions& config_options,
const Configurable* other,
std::string* mismatch) const override;
#ifndef ROCKSDB_LITE
// Gets the value of the option associated with the input name
// @see Configurable::GetOption for more details
Status GetOption(const ConfigOptions& config_options, const std::string& name,
std::string* value) const override;
#endif // ROCKSDB_LITE
protected:
// Given a name (e.g. rocksdb.my.type.opt), returns the short name (opt)
std::string GetOptionName(const std::string& long_name) const override;
#ifndef ROCKSDB_LITE
std::string SerializeOptions(const ConfigOptions& options,
const std::string& prefix) const override;
#endif // ROCKSDB_LITE
};
} // namespace ROCKSDB_NAMESPACE

@ -22,7 +22,7 @@
#include <string>
#include <unordered_map>
#include "rocksdb/configurable.h"
#include "rocksdb/customizable.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/status.h"
@ -613,7 +613,7 @@ extern TableFactory* NewCuckooTableFactory(
class RandomAccessFileReader;
// A base class for table factories.
class TableFactory : public Configurable {
class TableFactory : public Customizable {
public:
virtual ~TableFactory() override {}
@ -627,21 +627,7 @@ class TableFactory : public Configurable {
const std::string& id,
std::shared_ptr<TableFactory>* factory);
// The type of the table.
//
// The client of this package should switch to a new name whenever
// the table format implementation changes.
//
// Names starting with "rocksdb." are reserved and should not be used
// by any clients of this package.
virtual const char* Name() const = 0;
// Returns true if the class is an instance of the input name.
// This is typically determined by if the input name matches the
// name of this object.
virtual bool IsInstanceOf(const std::string& name) const {
return name == Name();
}
static const char* Type() { return "TableFactory"; }
// Returns a Table object table that can fetch data from file specified
// in parameter file. It's the caller's responsibility to make sure

@ -49,6 +49,7 @@ enum class OptionType {
kStruct,
kVector,
kConfigurable,
kCustomizable,
kUnknown,
};
@ -93,13 +94,14 @@ enum class OptionTypeFlags : uint32_t {
kCompareLoose = ConfigOptions::kSanityLevelLooselyCompatible,
kCompareExact = ConfigOptions::kSanityLevelExactMatch,
kMutable = 0x0100, // Option is mutable
kRawPointer = 0x0200, // The option is stored as a raw pointer
kShared = 0x0400, // The option is stored as a shared_ptr
kUnique = 0x0800, // The option is stored as a unique_ptr
kAllowNull = 0x1000, // The option can be null
kDontSerialize = 0x2000, // Don't serialize the option
kDontPrepare = 0x4000, // Don't prepare or sanitize this option
kMutable = 0x0100, // Option is mutable
kRawPointer = 0x0200, // The option is stored as a raw pointer
kShared = 0x0400, // The option is stored as a shared_ptr
kUnique = 0x0800, // The option is stored as a unique_ptr
kAllowNull = 0x1000, // The option can be null
kDontSerialize = 0x2000, // Don't serialize the option
kDontPrepare = 0x4000, // Don't prepare or sanitize this option
kStringNameOnly = 0x8000, // The option serializes to a name only
};
inline OptionTypeFlags operator|(const OptionTypeFlags &a,
@ -406,6 +408,103 @@ class OptionTypeInfo {
});
}
// Create a new std::shared_ptr<Customizable> OptionTypeInfo
// This function will call the T::CreateFromString method to create a new
// std::shared_ptr<T> object.
//
// @param offset The offset for the Customizable from the base pointer
// @param ovt How to verify this option
// @param flags, Extra flags specifying the behavior of this option
// @param _sfunc Optional function for serializing this option
// @param _efunc Optional function for comparing this option
template <typename T>
static OptionTypeInfo AsCustomSharedPtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags) {
return AsCustomSharedPtr<T>(offset, ovt, flags, nullptr, nullptr);
}
template <typename T>
static OptionTypeInfo AsCustomSharedPtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
return OptionTypeInfo(
offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kShared,
[](const ConfigOptions& opts, const std::string&,
const std::string& value, char* addr) {
auto* shared = reinterpret_cast<std::shared_ptr<T>*>(addr);
return T::CreateFromString(opts, value, shared);
},
serialize_func, equals_func);
}
// Create a new std::unique_ptr<Customizable> OptionTypeInfo
// This function will call the T::CreateFromString method to create a new
// std::unique_ptr<T> object.
//
// @param offset The offset for the Customizable from the base pointer
// @param ovt How to verify this option
// @param flags, Extra flags specifying the behavior of this option
// @param _sfunc Optional function for serializing this option
// @param _efunc Optional function for comparing this option
template <typename T>
static OptionTypeInfo AsCustomUniquePtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags) {
return AsCustomUniquePtr<T>(offset, ovt, flags, nullptr, nullptr);
}
template <typename T>
static OptionTypeInfo AsCustomUniquePtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
return OptionTypeInfo(
offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kUnique,
[](const ConfigOptions& opts, const std::string&,
const std::string& value, char* addr) {
auto* unique = reinterpret_cast<std::unique_ptr<T>*>(addr);
return T::CreateFromString(opts, value, unique);
},
serialize_func, equals_func);
}
// Create a new Customizable* OptionTypeInfo
// This function will call the T::CreateFromString method to create a new
// T object.
//
// @param _offset The offset for the Customizable from the base pointer
// @param ovt How to verify this option
// @param flags, Extra flags specifying the behavior of this option
// @param _sfunc Optional function for serializing this option
// @param _efunc Optional function for comparing this option
template <typename T>
static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt,
OptionTypeFlags flags) {
return AsCustomRawPtr<T>(offset, ovt, flags, nullptr, nullptr);
}
template <typename T>
static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
return OptionTypeInfo(
offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kRawPointer,
[](const ConfigOptions& opts, const std::string&,
const std::string& value, char* addr) {
auto** pointer = reinterpret_cast<T**>(addr);
return T::CreateFromString(opts, value, pointer);
},
serialize_func, equals_func);
}
bool IsEnabled(OptionTypeFlags otf) const { return (flags_ & otf) == otf; }
bool IsMutable() const { return IsEnabled(OptionTypeFlags::kMutable); }
@ -475,7 +574,12 @@ class OptionTypeInfo {
bool IsStruct() const { return (type_ == OptionType::kStruct); }
bool IsConfigurable() const { return (type_ == OptionType::kConfigurable); }
bool IsConfigurable() const {
return (type_ == OptionType::kConfigurable ||
type_ == OptionType::kCustomizable);
}
bool IsCustomizable() const { return (type_ == OptionType::kCustomizable); }
// Returns the underlying pointer for the type at base_addr
// The value returned is the underlying "raw" pointer, offset from base.
@ -660,6 +764,10 @@ Status ParseVector(const ConfigOptions& config_options,
result->clear();
Status status;
// Turn off ignore_unknown_objects so we can tell if the returned
// object is valid or not.
ConfigOptions copy = config_options;
copy.ignore_unsupported_options = false;
for (size_t start = 0, end = 0;
status.ok() && start < value.size() && end != std::string::npos;
start = end + 1) {
@ -667,10 +775,15 @@ Status ParseVector(const ConfigOptions& config_options,
status = OptionTypeInfo::NextToken(value, separator, start, &end, &token);
if (status.ok()) {
T elem;
status = elem_info.Parse(config_options, name, token,
reinterpret_cast<char*>(&elem));
status =
elem_info.Parse(copy, name, token, reinterpret_cast<char*>(&elem));
if (status.ok()) {
result->emplace_back(elem);
} else if (config_options.ignore_unsupported_options &&
status.IsNotSupported()) {
// If we were ignoring unsupported options and this one should be
// ignored, ignore it by setting the status to OK
status = Status::OK();
}
}
}

@ -567,31 +567,15 @@ static std::unordered_map<std::string, OptionTypeInfo>
}
return s;
}}},
{"table_factory",
{offset_of(&ColumnFamilyOptions::table_factory),
OptionType::kConfigurable, OptionVerificationType::kByName,
(OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose |
OptionTypeFlags::kDontPrepare),
// Creates a new TableFactory based on value
[](const ConfigOptions& opts, const std::string& /*name*/,
const std::string& value, char* addr) {
auto table_factory =
reinterpret_cast<std::shared_ptr<TableFactory>*>(addr);
return TableFactory::CreateFromString(opts, value, table_factory);
},
// Converts the TableFactory into its string representation
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const char* addr, std::string* value) {
const auto* table_factory =
reinterpret_cast<const std::shared_ptr<TableFactory>*>(addr);
*value = table_factory->get() ? table_factory->get()->Name()
: kNullptrString;
return Status::OK();
},
/* No equals function for table factories */ nullptr}},
{"table_factory", OptionTypeInfo::AsCustomSharedPtr<TableFactory>(
offset_of(&ColumnFamilyOptions::table_factory),
OptionVerificationType::kByName,
(OptionTypeFlags::kCompareLoose |
OptionTypeFlags::kStringNameOnly |
OptionTypeFlags::kDontPrepare))},
{"block_based_table_factory",
{offset_of(&ColumnFamilyOptions::table_factory),
OptionType::kConfigurable, OptionVerificationType::kAlias,
OptionType::kCustomizable, OptionVerificationType::kAlias,
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
// Parses the input value and creates a BlockBasedTableFactory
[](const ConfigOptions& opts, const std::string& name,
@ -623,7 +607,7 @@ static std::unordered_map<std::string, OptionTypeInfo>
}}},
{"plain_table_factory",
{offset_of(&ColumnFamilyOptions::table_factory),
OptionType::kConfigurable, OptionVerificationType::kAlias,
OptionType::kCustomizable, OptionVerificationType::kAlias,
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
// Parses the input value and creates a PlainTableFactory
[](const ConfigOptions& opts, const std::string& name,

@ -8,6 +8,7 @@
#include "logging/logging.h"
#include "options/configurable_helper.h"
#include "options/options_helper.h"
#include "rocksdb/customizable.h"
#include "rocksdb/status.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
@ -57,13 +58,9 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) {
}
}
}
#else
(void)opts;
#endif // ROCKSDB_LITE
if (status.ok()) {
auto inner = Inner();
if (inner != nullptr) {
status = inner->PrepareOptions(opts);
}
}
if (status.ok()) {
prepared_ = true;
}
@ -94,13 +91,10 @@ Status Configurable::ValidateOptions(const DBOptions& db_opts,
}
}
}
#else
(void)db_opts;
(void)cf_opts;
#endif // ROCKSDB_LITE
if (status.ok()) {
const auto inner = Inner();
if (inner != nullptr) {
status = inner->ValidateOptions(db_opts, cf_opts);
}
}
return status;
}
@ -116,12 +110,7 @@ const void* Configurable::GetOptionsPtr(const std::string& name) const {
return o.opt_ptr;
}
}
auto inner = Inner();
if (inner != nullptr) {
return inner->GetOptionsPtr(name);
} else {
return nullptr;
}
return nullptr;
}
std::string Configurable::GetOptionName(const std::string& opt_name) const {
@ -394,6 +383,23 @@ Status ConfigurableHelper::ConfigureOption(
if (opt_name == name) {
return configurable.ParseOption(config_options, opt_info, opt_name, value,
opt_ptr);
} else if (opt_info.IsCustomizable() &&
EndsWith(opt_name, ConfigurableHelper::kIdPropSuffix)) {
return configurable.ParseOption(config_options, opt_info, name, value,
opt_ptr);
} else if (opt_info.IsCustomizable()) {
Customizable* custom = opt_info.AsRawPointer<Customizable>(opt_ptr);
if (value.empty()) {
return Status::OK();
} else if (custom == nullptr || !StartsWith(name, custom->GetId() + ".")) {
return configurable.ParseOption(config_options, opt_info, name, value,
opt_ptr);
} else if (value.find("=") != std::string::npos) {
return custom->ConfigureFromString(config_options, value);
} else {
return custom->ConfigureOption(config_options, name, value);
}
} else if (opt_info.IsStruct() || opt_info.IsConfigurable()) {
return configurable.ParseOption(config_options, opt_info, name, value,
opt_ptr);
@ -403,6 +409,32 @@ Status ConfigurableHelper::ConfigureOption(
}
#endif // ROCKSDB_LITE
Status ConfigurableHelper::ConfigureNewObject(
const ConfigOptions& config_options_in, Configurable* object,
const std::string& id, const std::string& base_opts,
const std::unordered_map<std::string, std::string>& opts) {
if (object != nullptr) {
ConfigOptions config_options = config_options_in;
config_options.invoke_prepare_options = false;
if (!base_opts.empty()) {
#ifndef ROCKSDB_LITE
// Don't run prepare options on the base, as we would do that on the
// overlay opts instead
Status status = object->ConfigureFromString(config_options, base_opts);
if (!status.ok()) {
return status;
}
#endif // ROCKSDB_LITE
}
if (!opts.empty()) {
return object->ConfigureFromMap(config_options, opts);
}
} else if (!opts.empty()) { // No object but no map. This is OK
return Status::InvalidArgument("Cannot configure null object ", id);
}
return Status::OK();
}
//*******************************************************************************
//
// Methods for Converting Options into strings
@ -607,4 +639,43 @@ bool ConfigurableHelper::AreEquivalent(const ConfigOptions& config_options,
return true;
}
#endif // ROCKSDB_LITE
Status ConfigurableHelper::GetOptionsMap(
const std::string& value, std::string* id,
std::unordered_map<std::string, std::string>* props) {
return GetOptionsMap(value, "", id, props);
}
Status ConfigurableHelper::GetOptionsMap(
const std::string& value, const std::string& default_id, std::string* id,
std::unordered_map<std::string, std::string>* props) {
assert(id);
assert(props);
Status status;
if (value.empty() || value == kNullptrString) {
*id = default_id;
} else if (value.find('=') == std::string::npos) {
*id = value;
#ifndef ROCKSDB_LITE
} else {
status = StringToMap(value, props);
if (status.ok()) {
auto iter = props->find(ConfigurableHelper::kIdPropName);
if (iter != props->end()) {
*id = iter->second;
props->erase(iter);
} else if (default_id.empty()) { // Should this be an error??
status = Status::InvalidArgument("Name property is missing");
} else {
*id = default_id;
}
}
#else
} else {
*id = value;
props->clear();
#endif
}
return status;
}
} // namespace ROCKSDB_NAMESPACE

@ -20,6 +20,8 @@ namespace ROCKSDB_NAMESPACE {
// of configuring the objects.
class ConfigurableHelper {
public:
constexpr static const char* kIdPropName = "id";
constexpr static const char* kIdPropSuffix = ".id";
// Registers the input name with the options and associated map.
// When classes register their options in this manner, most of the
// functionality (excluding unknown options and validate/prepare) is
@ -75,6 +77,43 @@ class ConfigurableHelper {
const std::unordered_map<std::string, std::string>& options,
std::unordered_map<std::string, std::string>* unused);
// Helper method for configuring a new customizable object.
// If base_opts are set, this is the "default" options to use for the new
// object. Then any values in "new_opts" are applied to the object.
// Returns OK if the object could be successfully configured
// @return NotFound If any of the names in the base or new opts were not valid
// for this object.
// @return NotSupported If any of the names are valid but the object does
// not know how to convert the value. This can happen if, for example,
// there is some nested Configurable that cannot be created.
// @return InvalidArgument If any of the values cannot be successfully
// parsed.
static Status ConfigureNewObject(
const ConfigOptions& config_options, Configurable* object,
const std::string& id, const std::string& base_opts,
const std::unordered_map<std::string, std::string>& new_opts);
// Splits the input opt_value into the ID field and the remaining options.
// The input opt_value can be in the form of "name" or "name=value
// [;name=value]". The first form uses the "name" as an id with no options The
// latter form converts the input into a map of name=value pairs and sets "id"
// to the "id" value from the map.
// @param opt_value The value to split into id and options
// @param id The id field from the opt_value
// @param options The remaining name/value pairs from the opt_value
// @param default_id If specified and there is no id field in the map, this
// value is returned as the ID
// @return OK if the value was converted to a map succesfully and an ID was
// found.
// @return InvalidArgument if the value could not be converted to a map or
// there was or there is no id property in the map.
static Status GetOptionsMap(
const std::string& opt_value, std::string* id,
std::unordered_map<std::string, std::string>* options);
static Status GetOptionsMap(
const std::string& opt_value, const std::string& default_id,
std::string* id, std::unordered_map<std::string, std::string>* options);
#ifndef ROCKSDB_LITE
// Internal method to configure a set of options for this object.
// Classes may override this value to change its behavior.
@ -205,6 +244,7 @@ class ConfigurableHelper {
static const OptionTypeInfo* FindOption(
const std::vector<Configurable::RegisteredOptions>& options,
const std::string& name, std::string* opt_name, void** opt_ptr);
#endif // ROCKSDB_LITE
};

@ -79,29 +79,6 @@ class SimpleConfigurable : public TestConfigurable<Configurable> {
}; // End class SimpleConfigurable
static std::unordered_map<std::string, OptionTypeInfo> wrapped_option_info = {
#ifndef ROCKSDB_LITE
{"inner",
{0, OptionType::kConfigurable, OptionVerificationType::kNormal,
OptionTypeFlags::kShared}},
#endif // ROCKSDB_LITE
};
class WrappedConfigurable : public SimpleConfigurable {
public:
WrappedConfigurable(const std::string& name, unsigned char mode,
const std::shared_ptr<Configurable>& t)
: SimpleConfigurable(name, mode, &simple_option_info), inner_(t) {
ConfigurableHelper::RegisterOptions(*this, "WrappedOptions", &inner_,
&wrapped_option_info);
}
protected:
Configurable* Inner() const override { return inner_.get(); }
private:
std::shared_ptr<Configurable> inner_;
};
using ConfigTestFactoryFunc = std::function<Configurable*()>;
class ConfigurableTest : public testing::Test {
@ -607,17 +584,6 @@ static std::unordered_map<std::string, ConfigTestFactoryFunc> TestFactories = {
TestConfigMode::kSimpleMode |
TestConfigMode::kNestedMode);
}},
{"ThreeWay",
[]() {
std::shared_ptr<Configurable> child;
child.reset(
SimpleConfigurable::Create("child", TestConfigMode::kDefaultMode));
std::shared_ptr<Configurable> parent;
parent.reset(new WrappedConfigurable(
"parent", TestConfigMode::kDefaultMode, child));
return new WrappedConfigurable("master", TestConfigMode::kDefaultMode,
parent);
}},
{"ThreeDeep",
[]() {
Configurable* simple = SimpleConfigurable::Create(
@ -765,10 +731,6 @@ INSTANTIATE_TEST_CASE_P(
"pointer={int=22;string=pointer};"
"unique={int=33;string=unique};"
"shared={int=44;string=shared}"),
std::pair<std::string, std::string>("ThreeWay",
"int=11;bool=true;string=outer;"
"inner={int=22;string=parent;"
"inner={int=33;string=child}};"),
std::pair<std::string, std::string>("ThreeDeep",
"int=11;bool=true;string=outer;"
"unique={int=22;string=inner;"

@ -0,0 +1,77 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "rocksdb/customizable.h"
#include "options/configurable_helper.h"
#include "rocksdb/convenience.h"
#include "rocksdb/status.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
std::string Customizable::GetOptionName(const std::string& long_name) const {
const std::string& name = Name();
size_t name_len = name.size();
if (long_name.size() > name_len + 1 &&
long_name.compare(0, name_len, name) == 0 &&
long_name.at(name_len) == '.') {
return long_name.substr(name_len + 1);
} else {
return Configurable::GetOptionName(long_name);
}
}
#ifndef ROCKSDB_LITE
Status Customizable::GetOption(const ConfigOptions& config_options,
const std::string& opt_name,
std::string* value) const {
if (opt_name == ConfigurableHelper::kIdPropName) {
*value = GetId();
return Status::OK();
} else {
return Configurable::GetOption(config_options, opt_name, value);
}
}
std::string Customizable::SerializeOptions(const ConfigOptions& config_options,
const std::string& prefix) const {
std::string result;
std::string parent;
if (!config_options.IsShallow()) {
parent = Configurable::SerializeOptions(config_options, "");
}
if (parent.empty()) {
result = GetId();
} else {
result.append(prefix + ConfigurableHelper::kIdPropName + "=" + GetId() +
config_options.delimiter);
result.append(parent);
}
return result;
}
#endif // ROCKSDB_LITE
bool Customizable::AreEquivalent(const ConfigOptions& config_options,
const Configurable* other,
std::string* mismatch) const {
if (config_options.sanity_level > ConfigOptions::kSanityLevelNone &&
this != other) {
const Customizable* custom = reinterpret_cast<const Customizable*>(other);
if (GetId() != custom->GetId()) {
*mismatch = ConfigurableHelper::kIdPropName;
return false;
} else if (config_options.sanity_level >
ConfigOptions::kSanityLevelLooselyCompatible) {
bool matches =
Configurable::AreEquivalent(config_options, other, mismatch);
return matches;
}
}
return true;
}
} // namespace ROCKSDB_NAMESPACE

@ -0,0 +1,216 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <functional>
#include <memory>
#include <unordered_map>
#include "options/configurable_helper.h"
#include "options/options_helper.h"
#include "rocksdb/convenience.h"
#include "rocksdb/customizable.h"
#include "rocksdb/status.h"
#include "rocksdb/utilities/object_registry.h"
namespace ROCKSDB_NAMESPACE {
template <typename T>
using SharedFactoryFunc =
std::function<bool(const std::string&, std::shared_ptr<T>*)>;
template <typename T>
using UniqueFactoryFunc =
std::function<bool(const std::string&, std::unique_ptr<T>*)>;
template <typename T>
using StaticFactoryFunc = std::function<bool(const std::string&, T**)>;
// Creates a new shared Customizable object based on the input parameters.
// This method parses the input value to determine the type of instance to
// create. If there is an existing instance (in result) and it is the same type
// as the object being created, the existing configuration is stored and used as
// the default for the new object.
//
// The value parameter specified the instance class of the object to create.
// If it is a simple string (e.g. BlockBasedTable), then the instance will be
// created using the default settings. If the value is a set of name-value
// pairs, then the "id" value is used to determine the instance to create and
// the remaining parameters are used to configure the object. Id name-value
// pairs are specified, there must be an "id=value" pairing or an error will
// result.
//
// The config_options parameter controls the process and how errors are
// returned. If ignore_unknown_options=true, unknown values are ignored during
// the configuration If ignore_unsupported_options=true, unknown instance types
// are ignored If invoke_prepare_options=true, the resulting instance wll be
// initialized (via PrepareOptions
//
// @param config_options Controls how the instance is created and errors are
// handled
// @param value Either the simple name of the instance to create, or a set of
// name-value pairs to
// create and initailzie the object
// @param func Optional function to call to attempt to create an instance
// @param result The newly created instance.
template <typename T>
static Status LoadSharedObject(const ConfigOptions& config_options,
const std::string& value,
const SharedFactoryFunc<T>& func,
std::shared_ptr<T>* result) {
std::string id;
std::unordered_map<std::string, std::string> opt_map;
Status status = ConfigurableHelper::GetOptionsMap(value, &id, &opt_map);
if (!status.ok()) { // GetOptionsMap failed
return status;
}
std::string curr_opts;
#ifndef ROCKSDB_LITE
if (result->get() != nullptr && result->get()->GetId() == id) {
// Try to get the existing options, ignoring any errors
ConfigOptions embedded = config_options;
embedded.delimiter = ";";
result->get()->GetOptionString(embedded, &curr_opts).PermitUncheckedError();
}
#endif
if (func == nullptr || !func(id, result)) { // No factory, or it failed
if (id.empty() && opt_map.empty()) {
// No Id and no options. Clear the object
result->reset();
return Status::OK();
} else if (id.empty()) { // We have no Id but have options. Not good
return Status::NotSupported("Cannot reset object ", id);
} else {
#ifndef ROCKSDB_LITE
status = ObjectRegistry::NewInstance()->NewSharedObject(id, result);
#else
status = Status::NotSupported("Cannot load object in LITE mode ", id);
#endif
if (!status.ok()) {
if (config_options.ignore_unsupported_options) {
return Status::OK();
} else {
return status;
}
}
}
}
return ConfigurableHelper::ConfigureNewObject(config_options, result->get(),
id, curr_opts, opt_map);
}
// Creates a new unique customizable instance object based on the input
// parameters.
// @see LoadSharedObject for more information on the inner workings of this
// method.
//
// @param config_options Controls how the instance is created and errors are
// handled
// @param value Either the simple name of the instance to create, or a set of
// name-value pairs to
// create and initailzie the object
// @param func Optional function to call to attempt to create an instance
// @param result The newly created instance.
template <typename T>
static Status LoadUniqueObject(const ConfigOptions& config_options,
const std::string& value,
const UniqueFactoryFunc<T>& func,
std::unique_ptr<T>* result) {
std::string id;
std::unordered_map<std::string, std::string> opt_map;
Status status = ConfigurableHelper::GetOptionsMap(value, &id, &opt_map);
if (!status.ok()) { // GetOptionsMap failed
return status;
}
std::string curr_opts;
#ifndef ROCKSDB_LITE
if (result->get() != nullptr && result->get()->GetId() == id) {
// Try to get the existing options, ignoring any errors
ConfigOptions embedded = config_options;
embedded.delimiter = ";";
result->get()->GetOptionString(embedded, &curr_opts).PermitUncheckedError();
}
#endif
if (func == nullptr || !func(id, result)) { // No factory, or it failed
if (id.empty() && opt_map.empty()) {
// No Id and no options. Clear the object
result->reset();
return Status::OK();
} else if (id.empty()) { // We have no Id but have options. Not good
return Status::NotSupported("Cannot reset object ", id);
} else {
#ifndef ROCKSDB_LITE
status = ObjectRegistry::NewInstance()->NewUniqueObject(id, result);
#else
status = Status::NotSupported("Cannot load object in LITE mode ", id);
#endif // ROCKSDB_LITE
if (!status.ok()) {
if (config_options.ignore_unsupported_options) {
return Status::OK();
} else {
return status;
}
}
}
}
return ConfigurableHelper::ConfigureNewObject(config_options, result->get(),
id, curr_opts, opt_map);
}
// Creates a new static (raw pointer) customizable instance object based on the
// input parameters.
// @see LoadSharedObject for more information on the inner workings of this
// method.
//
// @param config_options Controls how the instance is created and errors are
// handled
// @param value Either the simple name of the instance to create, or a set of
// name-value pairs to
// create and initailzie the object
// @param func Optional function to call to attempt to create an instance
// @param result The newly created instance.
template <typename T>
static Status LoadStaticObject(const ConfigOptions& config_options,
const std::string& value,
const StaticFactoryFunc<T>& func, T** result) {
std::string id;
std::unordered_map<std::string, std::string> opt_map;
Status status = ConfigurableHelper::GetOptionsMap(value, &id, &opt_map);
if (!status.ok()) { // GetOptionsMap failed
return status;
}
std::string curr_opts;
#ifndef ROCKSDB_LITE
if (*result != nullptr && (*result)->GetId() == id) {
// Try to get the existing options, ignoring any errors
ConfigOptions embedded = config_options;
embedded.delimiter = ";";
(*result)->GetOptionString(embedded, &curr_opts).PermitUncheckedError();
}
#endif
if (func == nullptr || !func(id, result)) { // No factory, or it failed
if (id.empty() && opt_map.empty()) {
// No Id and no options. Clear the object
*result = nullptr;
return Status::OK();
} else if (id.empty()) { // We have no Id but have options. Not good
return Status::NotSupported("Cannot reset object ", id);
} else {
#ifndef ROCKSDB_LITE
status = ObjectRegistry::NewInstance()->NewStaticObject(id, result);
#else
status = Status::NotSupported("Cannot load object in LITE mode ", id);
#endif // ROCKSDB_LITE
if (!status.ok()) {
if (config_options.ignore_unsupported_options) {
return Status::OK();
} else {
return status;
}
}
}
}
return ConfigurableHelper::ConfigureNewObject(config_options, *result, id,
curr_opts, opt_map);
}
} // namespace ROCKSDB_NAMESPACE

@ -0,0 +1,625 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/customizable.h"
#include <cctype>
#include <cinttypes>
#include <cstring>
#include <unordered_map>
#include "options/configurable_helper.h"
#include "options/customizable_helper.h"
#include "options/options_helper.h"
#include "options/options_parser.h"
#include "rocksdb/convenience.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "table/mock_table.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#ifndef GFLAGS
bool FLAGS_enable_print = false;
#else
#include "util/gflags_compat.h"
using GFLAGS_NAMESPACE::ParseCommandLineFlags;
DEFINE_bool(enable_print, false, "Print options generated to console.");
#endif // GFLAGS
namespace ROCKSDB_NAMESPACE {
class StringLogger : public Logger {
public:
using Logger::Logv;
void Logv(const char* format, va_list ap) override {
char buffer[1000];
vsnprintf(buffer, sizeof(buffer), format, ap);
string_.append(buffer);
}
const std::string& str() const { return string_; }
void clear() { string_.clear(); }
private:
std::string string_;
};
class TestCustomizable : public Customizable {
public:
TestCustomizable(const std::string& name) : name_(name) {}
// Method to allow CheckedCast to work for this class
static const char* kClassName() {
return "TestCustomizable";
;
}
const char* Name() const override { return name_.c_str(); }
static const char* Type() { return "test.custom"; }
static Status CreateFromString(const ConfigOptions& opts,
const std::string& value,
std::unique_ptr<TestCustomizable>* result);
static Status CreateFromString(const ConfigOptions& opts,
const std::string& value,
std::shared_ptr<TestCustomizable>* result);
static Status CreateFromString(const ConfigOptions& opts,
const std::string& value,
TestCustomizable** result);
bool IsInstanceOf(const std::string& name) const override {
if (name == kClassName()) {
return true;
} else {
return Customizable::IsInstanceOf(name);
}
}
protected:
const std::string name_;
};
struct AOptions {
int i = 0;
bool b = false;
};
static std::unordered_map<std::string, OptionTypeInfo> a_option_info = {
#ifndef ROCKSDB_LITE
{"int",
{offsetof(struct AOptions, i), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"bool",
{offsetof(struct AOptions, b), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
class ACustomizable : public TestCustomizable {
public:
ACustomizable(const std::string& id) : TestCustomizable("A"), id_(id) {
ConfigurableHelper::RegisterOptions(*this, "A", &opts_, &a_option_info);
}
std::string GetId() const override { return id_; }
static const char* kClassName() { return "A"; }
private:
AOptions opts_;
const std::string id_;
};
#ifndef ROCKSDB_LITE
static int A_count = 0;
const FactoryFunc<TestCustomizable>& a_func =
ObjectLibrary::Default()->Register<TestCustomizable>(
"A.*",
[](const std::string& name, std::unique_ptr<TestCustomizable>* guard,
std::string* /* msg */) {
guard->reset(new ACustomizable(name));
A_count++;
return guard->get();
});
#endif // ROCKSDB_LITE
struct BOptions {
std::string s;
bool b = false;
};
static std::unordered_map<std::string, OptionTypeInfo> b_option_info = {
#ifndef ROCKSDB_LITE
{"string",
{offsetof(struct BOptions, s), OptionType::kString,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"bool",
{offsetof(struct BOptions, b), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
class BCustomizable : public TestCustomizable {
private:
public:
BCustomizable(const std::string& name) : TestCustomizable(name) {
ConfigurableHelper::RegisterOptions(*this, name, &opts_, &b_option_info);
}
static const char* kClassName() { return "B"; }
private:
BOptions opts_;
};
static bool LoadSharedB(const std::string& id,
std::shared_ptr<TestCustomizable>* result) {
if (id == "B") {
result->reset(new BCustomizable(id));
return true;
} else if (id.empty()) {
result->reset();
return true;
} else {
return false;
}
}
Status TestCustomizable::CreateFromString(
const ConfigOptions& config_options, const std::string& value,
std::shared_ptr<TestCustomizable>* result) {
return LoadSharedObject<TestCustomizable>(config_options, value, LoadSharedB,
result);
}
Status TestCustomizable::CreateFromString(
const ConfigOptions& config_options, const std::string& value,
std::unique_ptr<TestCustomizable>* result) {
return LoadUniqueObject<TestCustomizable>(
config_options, value,
[](const std::string& id, std::unique_ptr<TestCustomizable>* u) {
if (id == "B") {
u->reset(new BCustomizable(id));
return true;
} else if (id.empty()) {
u->reset();
return true;
} else {
return false;
}
},
result);
}
Status TestCustomizable::CreateFromString(const ConfigOptions& config_options,
const std::string& value,
TestCustomizable** result) {
return LoadStaticObject<TestCustomizable>(
config_options, value,
[](const std::string& id, TestCustomizable** ptr) {
if (id == "B") {
*ptr = new BCustomizable(id);
return true;
} else if (id.empty()) {
*ptr = nullptr;
return true;
} else {
return false;
}
},
result);
}
#ifndef ROCKSDB_LITE
const FactoryFunc<TestCustomizable>& s_func =
ObjectLibrary::Default()->Register<TestCustomizable>(
"S", [](const std::string& name,
std::unique_ptr<TestCustomizable>* /* guard */,
std::string* /* msg */) { return new BCustomizable(name); });
#endif // ROCKSDB_LITE
struct SimpleOptions {
bool b = true;
std::unique_ptr<TestCustomizable> cu;
std::shared_ptr<TestCustomizable> cs;
TestCustomizable* cp = nullptr;
};
static std::unordered_map<std::string, OptionTypeInfo> simple_option_info = {
#ifndef ROCKSDB_LITE
{"bool",
{offsetof(struct SimpleOptions, b), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"unique", OptionTypeInfo::AsCustomUniquePtr<TestCustomizable>(
offsetof(struct SimpleOptions, cu),
OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
{"shared", OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
offsetof(struct SimpleOptions, cs),
OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
{"pointer", OptionTypeInfo::AsCustomRawPtr<TestCustomizable>(
offsetof(struct SimpleOptions, cp),
OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
#endif // ROCKSDB_LITE
};
class SimpleConfigurable : public Configurable {
private:
SimpleOptions simple_;
public:
SimpleConfigurable() {
ConfigurableHelper::RegisterOptions(*this, "simple", &simple_,
&simple_option_info);
}
SimpleConfigurable(
const std::unordered_map<std::string, OptionTypeInfo>* map) {
ConfigurableHelper::RegisterOptions(*this, "simple", &simple_, map);
}
};
class CustomizableTest : public testing::Test {
public:
ConfigOptions config_options_;
};
#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
// Tests that a Customizable can be created by:
// - a simple name
// - a XXX.id option
// - a property with a name
TEST_F(CustomizableTest, CreateByNameTest) {
ObjectLibrary::Default()->Register<TestCustomizable>(
"TEST.*",
[](const std::string& name, std::unique_ptr<TestCustomizable>* guard,
std::string* /* msg */) {
guard->reset(new TestCustomizable(name));
return guard->get();
});
std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
SimpleOptions* simple = configurable->GetOptions<SimpleOptions>("simple");
ASSERT_NE(simple, nullptr);
ASSERT_OK(
configurable->ConfigureFromString(config_options_, "unique={id=TEST_1}"));
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(simple->cu->GetId(), "TEST_1");
ASSERT_OK(
configurable->ConfigureFromString(config_options_, "unique.id=TEST_2"));
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(simple->cu->GetId(), "TEST_2");
ASSERT_OK(
configurable->ConfigureFromString(config_options_, "unique=TEST_3"));
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(simple->cu->GetId(), "TEST_3");
}
TEST_F(CustomizableTest, ToStringTest) {
std::unique_ptr<TestCustomizable> custom(new TestCustomizable("test"));
ASSERT_EQ(custom->ToString(config_options_), "test");
}
TEST_F(CustomizableTest, SimpleConfigureTest) {
std::unordered_map<std::string, std::string> opt_map = {
{"unique", "id=A;int=1;bool=true"},
{"shared", "id=B;string=s"},
};
std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map));
SimpleOptions* simple = configurable->GetOptions<SimpleOptions>("simple");
ASSERT_NE(simple, nullptr);
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(simple->cu->GetId(), "A");
std::string opt_str;
std::string mismatch;
ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str));
std::unique_ptr<Configurable> copy(new SimpleConfigurable());
ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
ASSERT_TRUE(
configurable->AreEquivalent(config_options_, copy.get(), &mismatch));
}
static void GetMapFromProperties(
const std::string& props,
std::unordered_map<std::string, std::string>* map) {
std::istringstream iss(props);
std::unordered_map<std::string, std::string> copy_map;
std::string line;
map->clear();
for (int line_num = 0; std::getline(iss, line); line_num++) {
std::string name;
std::string value;
ASSERT_OK(
RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num));
(*map)[name] = value;
}
}
TEST_F(CustomizableTest, ConfigureFromPropsTest) {
std::unordered_map<std::string, std::string> opt_map = {
{"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"},
{"shared.id", "B"}, {"shared.B.string", "s"},
};
std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map));
SimpleOptions* simple = configurable->GetOptions<SimpleOptions>("simple");
ASSERT_NE(simple, nullptr);
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(simple->cu->GetId(), "A");
std::string opt_str;
std::string mismatch;
config_options_.delimiter = "\n";
std::unordered_map<std::string, std::string> props;
ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str));
GetMapFromProperties(opt_str, &props);
std::unique_ptr<Configurable> copy(new SimpleConfigurable());
ASSERT_OK(copy->ConfigureFromMap(config_options_, props));
ASSERT_TRUE(
configurable->AreEquivalent(config_options_, copy.get(), &mismatch));
}
TEST_F(CustomizableTest, ConfigureFromShortTest) {
std::unordered_map<std::string, std::string> opt_map = {
{"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"},
{"shared.id", "B"}, {"shared.B.string", "s"},
};
std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map));
SimpleOptions* simple = configurable->GetOptions<SimpleOptions>("simple");
ASSERT_NE(simple, nullptr);
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(simple->cu->GetId(), "A");
}
TEST_F(CustomizableTest, AreEquivalentOptionsTest) {
std::unordered_map<std::string, std::string> opt_map = {
{"unique", "id=A;int=1;bool=true"},
{"shared", "id=A;int=1;bool=true"},
};
std::string mismatch;
ConfigOptions config_options = config_options_;
config_options.invoke_prepare_options = false;
std::unique_ptr<Configurable> c1(new SimpleConfigurable());
std::unique_ptr<Configurable> c2(new SimpleConfigurable());
ASSERT_OK(c1->ConfigureFromMap(config_options, opt_map));
ASSERT_OK(c2->ConfigureFromMap(config_options, opt_map));
ASSERT_TRUE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
SimpleOptions* simple = c1->GetOptions<SimpleOptions>("simple");
ASSERT_TRUE(
simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch));
ASSERT_OK(simple->cu->ConfigureOption(config_options, "int", "2"));
ASSERT_FALSE(
simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch));
ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
ConfigOptions loosely = config_options;
loosely.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch));
ASSERT_TRUE(simple->cu->AreEquivalent(loosely, simple->cs.get(), &mismatch));
ASSERT_OK(c1->ConfigureOption(config_options, "shared", "id=B;string=3"));
ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch));
ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
ASSERT_FALSE(simple->cs->AreEquivalent(loosely, simple->cu.get(), &mismatch));
simple->cs.reset();
ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch));
ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
}
// Tests that we can initialize a customizable from its options
TEST_F(CustomizableTest, ConfigureStandaloneCustomTest) {
std::unique_ptr<TestCustomizable> base, copy;
auto registry = ObjectRegistry::NewInstance();
ASSERT_OK(registry->NewUniqueObject<TestCustomizable>("A", &base));
ASSERT_OK(registry->NewUniqueObject<TestCustomizable>("A", &copy));
ASSERT_OK(base->ConfigureFromString(config_options_, "int=33;bool=true"));
std::string opt_str;
std::string mismatch;
ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
}
// Tests that we fail appropriately if the pattern is not registered
TEST_F(CustomizableTest, BadNameTest) {
config_options_.ignore_unsupported_options = false;
std::unique_ptr<Configurable> c1(new SimpleConfigurable());
ASSERT_NOK(
c1->ConfigureFromString(config_options_, "unique.shared.id=bad name"));
config_options_.ignore_unsupported_options = true;
ASSERT_OK(
c1->ConfigureFromString(config_options_, "unique.shared.id=bad name"));
}
// Tests that we fail appropriately if a bad option is passed to the underlying
// configurable
TEST_F(CustomizableTest, BadOptionTest) {
std::unique_ptr<Configurable> c1(new SimpleConfigurable());
ConfigOptions ignore = config_options_;
ignore.ignore_unknown_options = true;
ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.int=11"));
ASSERT_NOK(c1->ConfigureFromString(config_options_, "shared={id=B;int=1}"));
ASSERT_OK(c1->ConfigureFromString(ignore, "shared={id=A;string=s}"));
ASSERT_NOK(c1->ConfigureFromString(config_options_, "B.int=11"));
ASSERT_OK(c1->ConfigureFromString(ignore, "B.int=11"));
ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.string=s"));
ASSERT_OK(c1->ConfigureFromString(ignore, "A.string=s"));
// Test as detached
ASSERT_NOK(
c1->ConfigureFromString(config_options_, "shared.id=A;A.string=b}"));
ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=A;A.string=s}"));
}
// Tests that different IDs lead to different objects
TEST_F(CustomizableTest, UniqueIdTest) {
std::unique_ptr<Configurable> base(new SimpleConfigurable());
ASSERT_OK(base->ConfigureFromString(config_options_,
"unique={id=A_1;int=1;bool=true}"));
SimpleOptions* simple = base->GetOptions<SimpleOptions>("simple");
ASSERT_NE(simple, nullptr);
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(simple->cu->GetId(), std::string("A_1"));
std::string opt_str;
std::string mismatch;
ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
std::unique_ptr<Configurable> copy(new SimpleConfigurable());
ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_OK(base->ConfigureFromString(config_options_,
"unique={id=A_2;int=1;bool=true}"));
ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_EQ(simple->cu->GetId(), std::string("A_2"));
}
TEST_F(CustomizableTest, IsInstanceOfTest) {
std::shared_ptr<TestCustomizable> tc = std::make_shared<ACustomizable>("A");
ASSERT_TRUE(tc->IsInstanceOf("A"));
ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable"));
ASSERT_FALSE(tc->IsInstanceOf("B"));
ASSERT_EQ(tc->CheckedCast<ACustomizable>(), tc.get());
ASSERT_EQ(tc->CheckedCast<TestCustomizable>(), tc.get());
ASSERT_EQ(tc->CheckedCast<BCustomizable>(), nullptr);
tc.reset(new BCustomizable("B"));
ASSERT_TRUE(tc->IsInstanceOf("B"));
ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable"));
ASSERT_FALSE(tc->IsInstanceOf("A"));
ASSERT_EQ(tc->CheckedCast<BCustomizable>(), tc.get());
ASSERT_EQ(tc->CheckedCast<TestCustomizable>(), tc.get());
ASSERT_EQ(tc->CheckedCast<ACustomizable>(), nullptr);
}
static std::unordered_map<std::string, OptionTypeInfo> inner_option_info = {
#ifndef ROCKSDB_LITE
{"inner",
OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
0, OptionVerificationType::kNormal, OptionTypeFlags::kStringNameOnly)}
#endif // ROCKSDB_LITE
};
class ShallowCustomizable : public Customizable {
public:
ShallowCustomizable() {
inner_ = std::make_shared<ACustomizable>("a");
ConfigurableHelper::RegisterOptions(*this, "inner", &inner_,
&inner_option_info);
};
static const char* kClassName() { return "shallow"; }
const char* Name() const override { return kClassName(); }
private:
std::shared_ptr<TestCustomizable> inner_;
};
TEST_F(CustomizableTest, TestStringDepth) {
ConfigOptions shallow = config_options_;
std::unique_ptr<Configurable> c(new ShallowCustomizable());
std::string opt_str;
shallow.depth = ConfigOptions::Depth::kDepthShallow;
ASSERT_OK(c->GetOptionString(shallow, &opt_str));
ASSERT_EQ(opt_str, "inner=a;");
shallow.depth = ConfigOptions::Depth::kDepthDetailed;
ASSERT_OK(c->GetOptionString(shallow, &opt_str));
ASSERT_NE(opt_str, "inner=a;");
}
// Tests that we only get a new customizable when it changes
TEST_F(CustomizableTest, NewCustomizableTest) {
std::unique_ptr<Configurable> base(new SimpleConfigurable());
A_count = 0;
ASSERT_OK(base->ConfigureFromString(config_options_,
"unique={id=A_1;int=1;bool=true}"));
SimpleOptions* simple = base->GetOptions<SimpleOptions>("simple");
ASSERT_NE(simple, nullptr);
ASSERT_NE(simple->cu, nullptr);
ASSERT_EQ(A_count, 1); // Created one A
ASSERT_OK(base->ConfigureFromString(config_options_,
"unique={id=A_1;int=1;bool=false}"));
ASSERT_EQ(A_count, 2); // Create another A_1
ASSERT_OK(base->ConfigureFromString(config_options_,
"unique={id=A_2;int=1;bool=false}"));
ASSERT_EQ(A_count, 3); // Created another A
ASSERT_OK(base->ConfigureFromString(config_options_, "unique="));
ASSERT_EQ(simple->cu, nullptr);
ASSERT_EQ(A_count, 3);
}
TEST_F(CustomizableTest, IgnoreUnknownObjects) {
ConfigOptions ignore = config_options_;
std::shared_ptr<TestCustomizable> shared;
std::unique_ptr<TestCustomizable> unique;
TestCustomizable* pointer = nullptr;
ignore.ignore_unsupported_options = false;
ASSERT_NOK(
LoadSharedObject<TestCustomizable>(ignore, "Unknown", nullptr, &shared));
ASSERT_NOK(
LoadUniqueObject<TestCustomizable>(ignore, "Unknown", nullptr, &unique));
ASSERT_NOK(
LoadStaticObject<TestCustomizable>(ignore, "Unknown", nullptr, &pointer));
ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr);
ignore.ignore_unsupported_options = true;
ASSERT_OK(
LoadSharedObject<TestCustomizable>(ignore, "Unknown", nullptr, &shared));
ASSERT_OK(
LoadUniqueObject<TestCustomizable>(ignore, "Unknown", nullptr, &unique));
ASSERT_OK(
LoadStaticObject<TestCustomizable>(ignore, "Unknown", nullptr, &pointer));
ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr);
ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown", nullptr,
&shared));
ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown", nullptr,
&unique));
ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown", nullptr,
&pointer));
ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr);
ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
nullptr, &shared));
ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
nullptr, &unique));
ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
nullptr, &pointer));
ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr);
}
TEST_F(CustomizableTest, FactoryFunctionTest) {
std::shared_ptr<TestCustomizable> shared;
std::unique_ptr<TestCustomizable> unique;
TestCustomizable* pointer = nullptr;
ConfigOptions ignore = config_options_;
ignore.ignore_unsupported_options = false;
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &shared));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &unique));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &pointer));
ASSERT_NE(shared.get(), nullptr);
ASSERT_NE(unique.get(), nullptr);
ASSERT_NE(pointer, nullptr);
delete pointer;
pointer = nullptr;
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &shared));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &unique));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &pointer));
ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr);
ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &shared));
ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &unique));
ASSERT_NOK(
TestCustomizable::CreateFromString(ignore, "option=bad", &pointer));
ASSERT_EQ(pointer, nullptr);
}
#endif // !ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
#ifdef GFLAGS
ParseCommandLineFlags(&argc, &argv, true);
#endif // GFLAGS
return RUN_ALL_TESTS();
}

@ -1062,6 +1062,19 @@ Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
return serialize_func_(config_options, opt_name, opt_addr, opt_value);
} else if (SerializeSingleOptionHelper(opt_addr, type_, opt_value)) {
return Status::OK();
} else if (IsCustomizable()) {
const Customizable* custom = AsRawPointer<Customizable>(opt_ptr);
if (custom == nullptr) {
*opt_value = kNullptrString;
} else if (IsEnabled(OptionTypeFlags::kStringNameOnly) &&
!config_options.IsDetailed()) {
*opt_value = custom->GetId();
} else {
ConfigOptions embedded = config_options;
embedded.delimiter = ";";
*opt_value = custom->ToString(embedded);
}
return Status::OK();
} else if (IsConfigurable()) {
const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
if (config != nullptr) {

@ -127,6 +127,7 @@ LIB_SOURCES = \
monitoring/thread_status_util_debug.cc \
options/cf_options.cc \
options/configurable.cc \
options/customizable.cc \
options/db_options.cc \
options/options.cc \
options/options_helper.cc \
@ -458,6 +459,7 @@ TEST_MAIN_SOURCES = \
monitoring/statistics_test.cc \
monitoring/stats_history_test.cc \
options/configurable_test.cc \
options/customizable_test.cc \
options/options_settable_test.cc \
options/options_test.cc \
table/block_based/block_based_filter_block_test.cc \

@ -46,6 +46,9 @@ class BlockBasedTableFactory : public TableFactory {
~BlockBasedTableFactory() {}
// Method to allow CheckedCast to work for this class
static const char* kClassName() { return kBlockBasedTableName(); }
const char* Name() const override { return kBlockBasedTableName(); }
using TableFactory::NewTableReader;

@ -56,6 +56,8 @@ class CuckooTableFactory : public TableFactory {
const CuckooTableOptions& table_option = CuckooTableOptions());
~CuckooTableFactory() {}
// Method to allow CheckedCast to work for this class
static const char* kClassName() { return kCuckooTableName(); }
const char* Name() const override { return kCuckooTableName(); }
using TableFactory::NewTableReader;

@ -156,6 +156,8 @@ class PlainTableFactory : public TableFactory {
explicit PlainTableFactory(
const PlainTableOptions& _table_options = PlainTableOptions());
// Method to allow CheckedCast to work for this class
static const char* kClassName() { return kPlainTableName(); }
const char* Name() const override { return kPlainTableName(); }
using TableFactory::NewTableReader;
Status NewTableReader(const ReadOptions& ro,

@ -3,6 +3,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "options/customizable_helper.h"
#include "rocksdb/convenience.h"
#include "rocksdb/table.h"
#include "table/block_based/block_based_table_factory.h"
@ -11,23 +12,9 @@
namespace ROCKSDB_NAMESPACE {
Status TableFactory::CreateFromString(const ConfigOptions& config_options_in,
const std::string& id,
std::shared_ptr<TableFactory>* factory) {
Status status;
std::string name = id;
std::string existing_opts;
ConfigOptions config_options = config_options_in;
if (factory->get() != nullptr && name == factory->get()->Name()) {
config_options.delimiter = ";";
status = factory->get()->GetOptionString(config_options, &existing_opts);
if (!status.ok()) {
return status;
}
}
static bool LoadFactory(const std::string& name,
std::shared_ptr<TableFactory>* factory) {
bool success = true;
if (name == TableFactory::kBlockBasedTableName()) {
factory->reset(new BlockBasedTableFactory());
#ifndef ROCKSDB_LITE
@ -37,14 +24,15 @@ Status TableFactory::CreateFromString(const ConfigOptions& config_options_in,
factory->reset(new CuckooTableFactory());
#endif // ROCKSDB_LITE
} else {
status = Status::NotSupported("Could not load table factory: ", name);
return status;
}
if (status.ok() && !existing_opts.empty()) {
config_options.invoke_prepare_options = false;
status = factory->get()->ConfigureFromString(config_options, existing_opts);
success = false;
}
return status;
return success;
}
Status TableFactory::CreateFromString(const ConfigOptions& config_options,
const std::string& value,
std::shared_ptr<TableFactory>* factory) {
return LoadSharedObject<TableFactory>(config_options, value, LoadFactory,
factory);
}
} // namespace ROCKSDB_NAMESPACE

Loading…
Cancel
Save