Remove FactoryFunc from LoadXXXObject (#11203)

Summary:
The primary purpose of the FactoryFunc was to support LITE mode where the ObjectRegistry was not available.  With the removal of LITE mode, the function was no longer required.

Note that the MergeOperator had some private classes defined in header files.  To gain access to their constructors (and name methods), the class definitions were moved into header files.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/11203

Reviewed By: cbi42

Differential Revision: D43160255

Pulled By: pdillinger

fbshipit-source-id: f3a465fd5d1a7049b73ecf31e4b8c3762f6dae6c
oxigraph-8.1.1
mrambacher 2 years ago committed by Facebook GitHub Bot
parent 25e1365227
commit b6640c3117
  1. 1
      HISTORY.md
  2. 3
      cache/cache.cc
  3. 3
      db/compaction/sst_partitioner.cc
  4. 2
      db/event_helpers.cc
  5. 5
      env/env.cc
  6. 5
      env/env_encryption.cc
  7. 2
      env/file_system.cc
  8. 45
      include/rocksdb/utilities/customizable_util.h
  9. 2
      monitoring/statistics.cc
  10. 112
      options/customizable_test.cc
  11. 20
      table/block_based/flush_block_policy.cc
  12. 13
      table/table_factory.cc
  13. 3
      util/file_checksum_helper.cc
  14. 8
      utilities/compaction_filters.cc
  15. 56
      utilities/merge_operators.cc
  16. 92
      utilities/merge_operators/max.cc
  17. 35
      utilities/merge_operators/max_operator.h
  18. 100
      utilities/merge_operators/put.cc
  19. 56
      utilities/merge_operators/put_operator.h
  20. 77
      utilities/merge_operators/uint64add.cc
  21. 35
      utilities/merge_operators/uint64add.h
  22. 2
      utilities/table_properties_collectors/compact_on_deletion_collector.cc
  23. 3
      utilities/wal_filter.cc

@ -21,6 +21,7 @@
* Remove deprecated Env::LoadEnv(). Use Env::CreateFromString() instead. * Remove deprecated Env::LoadEnv(). Use Env::CreateFromString() instead.
* Remove deprecated FileSystem::Load(). Use FileSystem::CreateFromString() instead. * Remove deprecated FileSystem::Load(). Use FileSystem::CreateFromString() instead.
* Removed the deprecated version of these utility functions and the corresponding Java bindings: `LoadOptionsFromFile`, `LoadLatestOptions`, `CheckOptionsCompatibility`. * Removed the deprecated version of these utility functions and the corresponding Java bindings: `LoadOptionsFromFile`, `LoadLatestOptions`, `CheckOptionsCompatibility`.
* Remove the FactoryFunc from the LoadObject method from the Customizable helper methods.
### Public API Changes ### Public API Changes
* Moved rarely-needed Cache class definition to new advanced_cache.h, and added a CacheWrapper class to advanced_cache.h. Minor changes to SimCache API definitions. * Moved rarely-needed Cache class definition to new advanced_cache.h, and added a CacheWrapper class to advanced_cache.h. Minor changes to SimCache API definitions.

3
cache/cache.cc vendored

@ -87,8 +87,7 @@ Status SecondaryCache::CreateFromString(
} }
return status; return status;
} else { } else {
return LoadSharedObject<SecondaryCache>(config_options, value, nullptr, return LoadSharedObject<SecondaryCache>(config_options, value, result);
result);
} }
} }

@ -78,7 +78,6 @@ Status SstPartitionerFactory::CreateFromString(
std::call_once(once, [&]() { std::call_once(once, [&]() {
RegisterSstPartitionerFactories(*(ObjectLibrary::Default().get()), ""); RegisterSstPartitionerFactories(*(ObjectLibrary::Default().get()), "");
}); });
return LoadSharedObject<SstPartitionerFactory>(options, value, nullptr, return LoadSharedObject<SstPartitionerFactory>(options, value, result);
result);
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -13,7 +13,7 @@ namespace ROCKSDB_NAMESPACE {
Status EventListener::CreateFromString(const ConfigOptions& config_options, Status EventListener::CreateFromString(const ConfigOptions& config_options,
const std::string& id, const std::string& id,
std::shared_ptr<EventListener>* result) { std::shared_ptr<EventListener>* result) {
return LoadSharedObject<EventListener>(config_options, id, nullptr, result); return LoadSharedObject<EventListener>(config_options, id, result);
} }
namespace { namespace {

5
env/env.cc vendored

@ -641,7 +641,7 @@ Status Env::CreateFromString(const ConfigOptions& config_options,
} else { } else {
RegisterSystemEnvs(); RegisterSystemEnvs();
Env* env = *result; Env* env = *result;
Status s = LoadStaticObject<Env>(config_options, value, nullptr, &env); Status s = LoadStaticObject<Env>(config_options, value, &env);
if (s.ok()) { if (s.ok()) {
*result = env; *result = env;
} }
@ -1227,8 +1227,7 @@ Status SystemClock::CreateFromString(const ConfigOptions& config_options,
std::call_once(once, [&]() { std::call_once(once, [&]() {
RegisterBuiltinSystemClocks(*(ObjectLibrary::Default().get()), ""); RegisterBuiltinSystemClocks(*(ObjectLibrary::Default().get()), "");
}); });
return LoadSharedObject<SystemClock>(config_options, value, nullptr, return LoadSharedObject<SystemClock>(config_options, value, result);
result);
} }
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -1332,15 +1332,14 @@ Status BlockCipher::CreateFromString(const ConfigOptions& config_options,
const std::string& value, const std::string& value,
std::shared_ptr<BlockCipher>* result) { std::shared_ptr<BlockCipher>* result) {
RegisterEncryptionBuiltins(); RegisterEncryptionBuiltins();
return LoadSharedObject<BlockCipher>(config_options, value, nullptr, result); return LoadSharedObject<BlockCipher>(config_options, value, result);
} }
Status EncryptionProvider::CreateFromString( Status EncryptionProvider::CreateFromString(
const ConfigOptions& config_options, const std::string& value, const ConfigOptions& config_options, const std::string& value,
std::shared_ptr<EncryptionProvider>* result) { std::shared_ptr<EncryptionProvider>* result) {
RegisterEncryptionBuiltins(); RegisterEncryptionBuiltins();
return LoadSharedObject<EncryptionProvider>(config_options, value, nullptr, return LoadSharedObject<EncryptionProvider>(config_options, value, result);
result);
} }

@ -91,7 +91,7 @@ Status FileSystem::CreateFromString(const ConfigOptions& config_options,
std::call_once(once, [&]() { std::call_once(once, [&]() {
RegisterBuiltinFileSystems(*(ObjectLibrary::Default().get()), ""); RegisterBuiltinFileSystems(*(ObjectLibrary::Default().get()), "");
}); });
return LoadSharedObject<FileSystem>(config_options, value, nullptr, result); return LoadSharedObject<FileSystem>(config_options, value, result);
} }
} }

@ -13,7 +13,6 @@
// for more information on how to develop and use customizable objects // for more information on how to develop and use customizable objects
#pragma once #pragma once
#include <functional>
#include <memory> #include <memory>
#include <unordered_map> #include <unordered_map>
@ -24,24 +23,6 @@
#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/object_registry.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// The FactoryFunc functions are used to create a new customizable object
// without going through the ObjectRegistry. This methodology is especially
// useful in LITE mode, where there is no ObjectRegistry. The methods take
// in an ID of the object to create and a pointer to store the created object.
// If the factory successfully recognized the input ID, the method should return
// success; otherwise false should be returned. On success, the object
// parameter contains the new object.
template <typename T>
using SharedFactoryFunc =
std::function<bool(const std::string&, std::shared_ptr<T>*)>;
template <typename T>
using UniqueFactoryFunc =
std::function<bool(const std::string&, std::unique_ptr<T>*)>;
template <typename T>
using StaticFactoryFunc = std::function<bool(const std::string&, T**)>;
// Creates a new shared customizable instance object based on the // Creates a new shared customizable instance object based on the
// input parameters using the object registry. // input parameters using the object registry.
// //
@ -156,12 +137,10 @@ static Status NewManagedObject(
// handled // handled
// @param value Either the simple name of the instance to create, or a set of // @param value Either the simple name of the instance to create, or a set of
// name-value pairs to create and initailize the object // name-value pairs to create and initailize the object
// @param func Optional function to call to attempt to create an instance
// @param result The newly created instance. // @param result The newly created instance.
template <typename T> template <typename T>
static Status LoadSharedObject(const ConfigOptions& config_options, static Status LoadSharedObject(const ConfigOptions& config_options,
const std::string& value, const std::string& value,
const SharedFactoryFunc<T>& func,
std::shared_ptr<T>* result) { std::shared_ptr<T>* result) {
std::string id; std::string id;
std::unordered_map<std::string, std::string> opt_map; std::unordered_map<std::string, std::string> opt_map;
@ -170,12 +149,8 @@ static Status LoadSharedObject(const ConfigOptions& config_options,
value, &id, &opt_map); value, &id, &opt_map);
if (!status.ok()) { // GetOptionsMap failed if (!status.ok()) { // GetOptionsMap failed
return status; return status;
} else if (func == nullptr ||
!func(id, result)) { // No factory, or it failed
return NewSharedObject(config_options, id, opt_map, result);
} else { } else {
return Customizable::ConfigureNewObject(config_options, result->get(), return NewSharedObject(config_options, id, opt_map, result);
opt_map);
} }
} }
@ -204,7 +179,6 @@ static Status LoadSharedObject(const ConfigOptions& config_options,
// handled // handled
// @param value Either the simple name of the instance to create, or a set of // @param value Either the simple name of the instance to create, or a set of
// name-value pairs to create and initailize the object // name-value pairs to create and initailize the object
// @param func Optional function to call to attempt to create an instance
// @param result The newly created instance. // @param result The newly created instance.
template <typename T> template <typename T>
static Status LoadManagedObject(const ConfigOptions& config_options, static Status LoadManagedObject(const ConfigOptions& config_options,
@ -270,12 +244,10 @@ static Status NewUniqueObject(
// handled // handled
// @param value Either the simple name of the instance to create, or a set of // @param value Either the simple name of the instance to create, or a set of
// name-value pairs to create and initailize the object // name-value pairs to create and initailize the object
// @param func Optional function to call to attempt to create an instance
// @param result The newly created instance. // @param result The newly created instance.
template <typename T> template <typename T>
static Status LoadUniqueObject(const ConfigOptions& config_options, static Status LoadUniqueObject(const ConfigOptions& config_options,
const std::string& value, const std::string& value,
const UniqueFactoryFunc<T>& func,
std::unique_ptr<T>* result) { std::unique_ptr<T>* result) {
std::string id; std::string id;
std::unordered_map<std::string, std::string> opt_map; std::unordered_map<std::string, std::string> opt_map;
@ -283,12 +255,8 @@ static Status LoadUniqueObject(const ConfigOptions& config_options,
value, &id, &opt_map); value, &id, &opt_map);
if (!status.ok()) { // GetOptionsMap failed if (!status.ok()) { // GetOptionsMap failed
return status; return status;
} else if (func == nullptr ||
!func(id, result)) { // No factory, or it failed
return NewUniqueObject(config_options, id, opt_map, result);
} else { } else {
return Customizable::ConfigureNewObject(config_options, result->get(), return NewUniqueObject(config_options, id, opt_map, result);
opt_map);
} }
} }
@ -337,23 +305,18 @@ static Status NewStaticObject(
// handled // handled
// @param value Either the simple name of the instance to create, or a set of // @param value Either the simple name of the instance to create, or a set of
// name-value pairs to create and initailize the object // name-value pairs to create and initailize the object
// @param func Optional function to call to attempt to create an instance
// @param result The newly created instance. // @param result The newly created instance.
template <typename T> template <typename T>
static Status LoadStaticObject(const ConfigOptions& config_options, static Status LoadStaticObject(const ConfigOptions& config_options,
const std::string& value, const std::string& value, T** result) {
const StaticFactoryFunc<T>& func, T** result) {
std::string id; std::string id;
std::unordered_map<std::string, std::string> opt_map; std::unordered_map<std::string, std::string> opt_map;
Status status = Customizable::GetOptionsMap(config_options, *result, value, Status status = Customizable::GetOptionsMap(config_options, *result, value,
&id, &opt_map); &id, &opt_map);
if (!status.ok()) { // GetOptionsMap failed if (!status.ok()) { // GetOptionsMap failed
return status; return status;
} else if (func == nullptr ||
!func(id, result)) { // No factory, or it failed
return NewStaticObject(config_options, id, opt_map, result);
} else { } else {
return Customizable::ConfigureNewObject(config_options, *result, opt_map); return NewStaticObject(config_options, id, opt_map, result);
} }
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -300,7 +300,7 @@ Status Statistics::CreateFromString(const ConfigOptions& config_options,
} else if (id == kNullptrString) { } else if (id == kNullptrString) {
result->reset(); result->reset();
} else { } else {
s = LoadSharedObject<Statistics>(config_options, id, nullptr, result); s = LoadSharedObject<Statistics>(config_options, id, result);
} }
return s; return s;
} }

@ -160,19 +160,6 @@ class BCustomizable : public TestCustomizable {
BOptions opts_; BOptions opts_;
}; };
static bool LoadSharedB(const std::string& id,
std::shared_ptr<TestCustomizable>* result) {
if (id == "B") {
result->reset(new BCustomizable(id));
return true;
} else if (id.empty()) {
result->reset();
return true;
} else {
return false;
}
}
static int A_count = 0; static int A_count = 0;
static int RegisterCustomTestObjects(ObjectLibrary& library, static int RegisterCustomTestObjects(ObjectLibrary& library,
const std::string& /*arg*/) { const std::string& /*arg*/) {
@ -184,6 +171,12 @@ static int RegisterCustomTestObjects(ObjectLibrary& library,
A_count++; A_count++;
return guard->get(); return guard->get();
}); });
library.AddFactory<TestCustomizable>(
"B", [](const std::string& name, std::unique_ptr<TestCustomizable>* guard,
std::string* /* msg */) {
guard->reset(new BCustomizable(name));
return guard->get();
});
library.AddFactory<TestCustomizable>( library.AddFactory<TestCustomizable>(
"S", [](const std::string& name, "S", [](const std::string& name,
@ -252,46 +245,19 @@ static void GetMapFromProperties(
Status TestCustomizable::CreateFromString( Status TestCustomizable::CreateFromString(
const ConfigOptions& config_options, const std::string& value, const ConfigOptions& config_options, const std::string& value,
std::shared_ptr<TestCustomizable>* result) { std::shared_ptr<TestCustomizable>* result) {
return LoadSharedObject<TestCustomizable>(config_options, value, LoadSharedB, return LoadSharedObject<TestCustomizable>(config_options, value, result);
result);
} }
Status TestCustomizable::CreateFromString( Status TestCustomizable::CreateFromString(
const ConfigOptions& config_options, const std::string& value, const ConfigOptions& config_options, const std::string& value,
std::unique_ptr<TestCustomizable>* result) { std::unique_ptr<TestCustomizable>* result) {
return LoadUniqueObject<TestCustomizable>( return LoadUniqueObject<TestCustomizable>(config_options, value, result);
config_options, value,
[](const std::string& id, std::unique_ptr<TestCustomizable>* u) {
if (id == "B") {
u->reset(new BCustomizable(id));
return true;
} else if (id.empty()) {
u->reset();
return true;
} else {
return false;
}
},
result);
} }
Status TestCustomizable::CreateFromString(const ConfigOptions& config_options, Status TestCustomizable::CreateFromString(const ConfigOptions& config_options,
const std::string& value, const std::string& value,
TestCustomizable** result) { TestCustomizable** result) {
return LoadStaticObject<TestCustomizable>( return LoadStaticObject<TestCustomizable>(config_options, value, result);
config_options, value,
[](const std::string& id, TestCustomizable** ptr) {
if (id == "B") {
*ptr = new BCustomizable(id);
return true;
} else if (id.empty()) {
*ptr = nullptr;
return true;
} else {
return false;
}
},
result);
} }
class CustomizableTest : public testing::Test { class CustomizableTest : public testing::Test {
@ -980,70 +946,34 @@ TEST_F(CustomizableTest, IgnoreUnknownObjects) {
std::unique_ptr<TestCustomizable> unique; std::unique_ptr<TestCustomizable> unique;
TestCustomizable* pointer = nullptr; TestCustomizable* pointer = nullptr;
ignore.ignore_unsupported_options = false; ignore.ignore_unsupported_options = false;
ASSERT_NOK( ASSERT_NOK(LoadSharedObject<TestCustomizable>(ignore, "Unknown", &shared));
LoadSharedObject<TestCustomizable>(ignore, "Unknown", nullptr, &shared)); ASSERT_NOK(LoadUniqueObject<TestCustomizable>(ignore, "Unknown", &unique));
ASSERT_NOK( ASSERT_NOK(LoadStaticObject<TestCustomizable>(ignore, "Unknown", &pointer));
LoadUniqueObject<TestCustomizable>(ignore, "Unknown", nullptr, &unique));
ASSERT_NOK(
LoadStaticObject<TestCustomizable>(ignore, "Unknown", nullptr, &pointer));
ASSERT_EQ(shared.get(), nullptr); ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr); ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr); ASSERT_EQ(pointer, nullptr);
ignore.ignore_unsupported_options = true; ignore.ignore_unsupported_options = true;
ASSERT_OK( ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "Unknown", &shared));
LoadSharedObject<TestCustomizable>(ignore, "Unknown", nullptr, &shared)); ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "Unknown", &unique));
ASSERT_OK( ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "Unknown", &pointer));
LoadUniqueObject<TestCustomizable>(ignore, "Unknown", nullptr, &unique));
ASSERT_OK(
LoadStaticObject<TestCustomizable>(ignore, "Unknown", nullptr, &pointer));
ASSERT_EQ(shared.get(), nullptr); ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr); ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr); ASSERT_EQ(pointer, nullptr);
ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown", nullptr, ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown", &shared));
&shared)); ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown", &unique));
ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown", nullptr, ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown", &pointer));
&unique));
ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown", nullptr,
&pointer));
ASSERT_EQ(shared.get(), nullptr); ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr); ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr); ASSERT_EQ(pointer, nullptr);
ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown;option=bad", ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
nullptr, &shared)); &shared));
ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown;option=bad", ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
nullptr, &unique)); &unique));
ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown;option=bad", ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
nullptr, &pointer)); &pointer));
ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr);
}
TEST_F(CustomizableTest, FactoryFunctionTest) {
std::shared_ptr<TestCustomizable> shared;
std::unique_ptr<TestCustomizable> unique;
TestCustomizable* pointer = nullptr;
ConfigOptions ignore = config_options_;
ignore.ignore_unsupported_options = false;
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &shared));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &unique));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &pointer));
ASSERT_NE(shared.get(), nullptr);
ASSERT_NE(unique.get(), nullptr);
ASSERT_NE(pointer, nullptr);
delete pointer;
pointer = nullptr;
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "id=", &shared));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "id=", &unique));
ASSERT_OK(TestCustomizable::CreateFromString(ignore, "id=", &pointer));
ASSERT_EQ(shared.get(), nullptr); ASSERT_EQ(shared.get(), nullptr);
ASSERT_EQ(unique.get(), nullptr); ASSERT_EQ(unique.get(), nullptr);
ASSERT_EQ(pointer, nullptr); ASSERT_EQ(pointer, nullptr);
ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &shared));
ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &unique));
ASSERT_NOK(
TestCustomizable::CreateFromString(ignore, "option=bad", &pointer));
ASSERT_EQ(pointer, nullptr);
} }
TEST_F(CustomizableTest, URLFactoryTest) { TEST_F(CustomizableTest, URLFactoryTest) {

@ -110,16 +110,6 @@ static int RegisterFlushBlockPolicyFactories(ObjectLibrary& library,
return 2; return 2;
} }
static bool LoadFlushPolicyFactory(
const std::string& id, std::shared_ptr<FlushBlockPolicyFactory>* result) {
if (id.empty()) {
result->reset(new FlushBlockBySizePolicyFactory());
} else {
return false;
}
return true;
}
FlushBlockBySizePolicyFactory::FlushBlockBySizePolicyFactory() FlushBlockBySizePolicyFactory::FlushBlockBySizePolicyFactory()
: FlushBlockPolicyFactory() {} : FlushBlockPolicyFactory() {}
@ -130,7 +120,13 @@ Status FlushBlockPolicyFactory::CreateFromString(
std::call_once(once, [&]() { std::call_once(once, [&]() {
RegisterFlushBlockPolicyFactories(*(ObjectLibrary::Default().get()), ""); RegisterFlushBlockPolicyFactories(*(ObjectLibrary::Default().get()), "");
}); });
return LoadSharedObject<FlushBlockPolicyFactory>(
config_options, value, LoadFlushPolicyFactory, factory); if (value.empty()) {
factory->reset(new FlushBlockBySizePolicyFactory());
return Status::OK();
} else {
return LoadSharedObject<FlushBlockPolicyFactory>(config_options, value,
factory);
}
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -43,21 +43,10 @@ static void RegisterTableFactories(const std::string& /*arg*/) {
}); });
} }
static bool LoadFactory(const std::string& name,
std::shared_ptr<TableFactory>* factory) {
if (name == TableFactory::kBlockBasedTableName()) {
factory->reset(new BlockBasedTableFactory());
return true;
} else {
return false;
}
}
Status TableFactory::CreateFromString(const ConfigOptions& config_options, Status TableFactory::CreateFromString(const ConfigOptions& config_options,
const std::string& value, const std::string& value,
std::shared_ptr<TableFactory>* factory) { std::shared_ptr<TableFactory>* factory) {
RegisterTableFactories(""); RegisterTableFactories("");
return LoadSharedObject<TableFactory>(config_options, value, LoadFactory, return LoadSharedObject<TableFactory>(config_options, value, factory);
factory);
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -160,8 +160,7 @@ Status FileChecksumGenFactory::CreateFromString(
*result = GetFileChecksumGenCrc32cFactory(); *result = GetFileChecksumGenCrc32cFactory();
return Status::OK(); return Status::OK();
} else { } else {
Status s = LoadSharedObject<FileChecksumGenFactory>(options, value, nullptr, Status s = LoadSharedObject<FileChecksumGenFactory>(options, value, result);
result);
return s; return s;
} }
} }

@ -32,8 +32,8 @@ Status CompactionFilter::CreateFromString(const ConfigOptions& config_options,
RegisterBuiltinCompactionFilters(*(ObjectLibrary::Default().get()), ""); RegisterBuiltinCompactionFilters(*(ObjectLibrary::Default().get()), "");
}); });
CompactionFilter* filter = const_cast<CompactionFilter*>(*result); CompactionFilter* filter = const_cast<CompactionFilter*>(*result);
Status status = LoadStaticObject<CompactionFilter>(config_options, value, Status status =
nullptr, &filter); LoadStaticObject<CompactionFilter>(config_options, value, &filter);
if (status.ok()) { if (status.ok()) {
*result = const_cast<CompactionFilter*>(filter); *result = const_cast<CompactionFilter*>(filter);
} }
@ -45,8 +45,8 @@ Status CompactionFilterFactory::CreateFromString(
std::shared_ptr<CompactionFilterFactory>* result) { std::shared_ptr<CompactionFilterFactory>* result) {
// Currently there are no builtin CompactionFilterFactories. // Currently there are no builtin CompactionFilterFactories.
// If any are introduced, they need to be registered here. // If any are introduced, they need to be registered here.
Status status = LoadSharedObject<CompactionFilterFactory>( Status status =
config_options, value, nullptr, result); LoadSharedObject<CompactionFilterFactory>(config_options, value, result);
return status; return status;
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -12,30 +12,14 @@
#include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/object_registry.h"
#include "utilities/merge_operators/bytesxor.h" #include "utilities/merge_operators/bytesxor.h"
#include "utilities/merge_operators/max_operator.h"
#include "utilities/merge_operators/put_operator.h"
#include "utilities/merge_operators/sortlist.h" #include "utilities/merge_operators/sortlist.h"
#include "utilities/merge_operators/string_append/stringappend.h" #include "utilities/merge_operators/string_append/stringappend.h"
#include "utilities/merge_operators/string_append/stringappend2.h" #include "utilities/merge_operators/string_append/stringappend2.h"
#include "utilities/merge_operators/uint64add.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
static bool LoadMergeOperator(const std::string& id,
std::shared_ptr<MergeOperator>* result) {
bool success = true;
// TODO: Hook the "name" up to the actual Name() of the MergeOperators?
// Requires these classes be moved into a header file...
if (id == "put" || id == "PutOperator") {
*result = MergeOperators::CreatePutOperator();
} else if (id == "put_v1") {
*result = MergeOperators::CreateDeprecatedPutOperator();
} else if (id == "uint64add" || id == "UInt64AddOperator") {
*result = MergeOperators::CreateUInt64AddOperator();
} else if (id == "max" || id == "MaxOperator") {
*result = MergeOperators::CreateMaxOperator();
} else {
success = false;
}
return success;
}
static int RegisterBuiltinMergeOperators(ObjectLibrary& library, static int RegisterBuiltinMergeOperators(ObjectLibrary& library,
const std::string& /*arg*/) { const std::string& /*arg*/) {
size_t num_types; size_t num_types;
@ -71,6 +55,37 @@ static int RegisterBuiltinMergeOperators(ObjectLibrary& library,
guard->reset(new BytesXOROperator()); guard->reset(new BytesXOROperator());
return guard->get(); return guard->get();
}); });
library.AddFactory<MergeOperator>(
ObjectLibrary::PatternEntry(UInt64AddOperator::kClassName())
.AnotherName(UInt64AddOperator::kNickName()),
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard,
std::string* /*errmsg*/) {
guard->reset(new UInt64AddOperator());
return guard->get();
});
library.AddFactory<MergeOperator>(
ObjectLibrary::PatternEntry(MaxOperator::kClassName())
.AnotherName(MaxOperator::kNickName()),
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard,
std::string* /*errmsg*/) {
guard->reset(new MaxOperator());
return guard->get();
});
library.AddFactory<MergeOperator>(
ObjectLibrary::PatternEntry(PutOperatorV2::kClassName())
.AnotherName(PutOperatorV2::kNickName()),
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard,
std::string* /*errmsg*/) {
guard->reset(new PutOperatorV2());
return guard->get();
});
library.AddFactory<MergeOperator>(
ObjectLibrary::PatternEntry(PutOperator::kNickName()),
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard,
std::string* /*errmsg*/) {
guard->reset(new PutOperator());
return guard->get();
});
return static_cast<int>(library.GetFactoryCount(&num_types)); return static_cast<int>(library.GetFactoryCount(&num_types));
} }
@ -82,8 +97,7 @@ Status MergeOperator::CreateFromString(const ConfigOptions& config_options,
std::call_once(once, [&]() { std::call_once(once, [&]() {
RegisterBuiltinMergeOperators(*(ObjectLibrary::Default().get()), ""); RegisterBuiltinMergeOperators(*(ObjectLibrary::Default().get()), "");
}); });
return LoadSharedObject<MergeOperator>(config_options, value, return LoadSharedObject<MergeOperator>(config_options, value, result);
LoadMergeOperator, result);
} }
std::shared_ptr<MergeOperator> MergeOperators::CreateFromStringId( std::shared_ptr<MergeOperator> MergeOperators::CreateFromStringId(

@ -8,71 +8,55 @@
#include "rocksdb/merge_operator.h" #include "rocksdb/merge_operator.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "utilities/merge_operators.h" #include "utilities/merge_operators.h"
#include "utilities/merge_operators/max_operator.h"
using ROCKSDB_NAMESPACE::Logger; namespace ROCKSDB_NAMESPACE {
using ROCKSDB_NAMESPACE::MergeOperator;
using ROCKSDB_NAMESPACE::Slice;
namespace { // anonymous namespace
// Merge operator that picks the maximum operand, Comparison is based on
// Slice::compare
class MaxOperator : public MergeOperator {
public:
bool FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const override {
Slice& max = merge_out->existing_operand;
if (merge_in.existing_value) {
max = Slice(merge_in.existing_value->data(),
merge_in.existing_value->size());
} else if (max.data() == nullptr) {
max = Slice();
}
for (const auto& op : merge_in.operand_list) {
if (max.compare(op) < 0) {
max = op;
}
}
return true; bool MaxOperator::FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const {
Slice& max = merge_out->existing_operand;
if (merge_in.existing_value) {
max =
Slice(merge_in.existing_value->data(), merge_in.existing_value->size());
} else if (max.data() == nullptr) {
max = Slice();
} }
bool PartialMerge(const Slice& /*key*/, const Slice& left_operand, for (const auto& op : merge_in.operand_list) {
const Slice& right_operand, std::string* new_value, if (max.compare(op) < 0) {
Logger* /*logger*/) const override { max = op;
if (left_operand.compare(right_operand) >= 0) {
new_value->assign(left_operand.data(), left_operand.size());
} else {
new_value->assign(right_operand.data(), right_operand.size());
} }
return true;
} }
bool PartialMergeMulti(const Slice& /*key*/, return true;
const std::deque<Slice>& operand_list, }
std::string* new_value,
Logger* /*logger*/) const override {
Slice max;
for (const auto& operand : operand_list) {
if (max.compare(operand) < 0) {
max = operand;
}
}
new_value->assign(max.data(), max.size()); bool MaxOperator::PartialMerge(const Slice& /*key*/, const Slice& left_operand,
return true; const Slice& right_operand,
std::string* new_value,
Logger* /*logger*/) const {
if (left_operand.compare(right_operand) >= 0) {
new_value->assign(left_operand.data(), left_operand.size());
} else {
new_value->assign(right_operand.data(), right_operand.size());
} }
return true;
}
static const char* kClassName() { return "MaxOperator"; } bool MaxOperator::PartialMergeMulti(const Slice& /*key*/,
static const char* kNickName() { return "max"; } const std::deque<Slice>& operand_list,
const char* Name() const override { return kClassName(); } std::string* new_value,
const char* NickName() const override { return kNickName(); } Logger* /*logger*/) const {
}; Slice max;
for (const auto& operand : operand_list) {
} // end of anonymous namespace if (max.compare(operand) < 0) {
max = operand;
}
}
namespace ROCKSDB_NAMESPACE { new_value->assign(max.data(), max.size());
return true;
}
std::shared_ptr<MergeOperator> MergeOperators::CreateMaxOperator() { std::shared_ptr<MergeOperator> MergeOperators::CreateMaxOperator() {
return std::make_shared<MaxOperator>(); return std::make_shared<MaxOperator>();

@ -0,0 +1,35 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Merge operator that picks the maximum operand, Comparison is based on
// Slice::compare
#pragma once
#include "rocksdb/merge_operator.h"
namespace ROCKSDB_NAMESPACE {
class Logger;
class Slice;
class MaxOperator : public MergeOperator {
public:
static const char* kClassName() { return "MaxOperator"; }
static const char* kNickName() { return "max"; }
const char* Name() const override { return kClassName(); }
const char* NickName() const override { return kNickName(); }
bool FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const override;
bool PartialMerge(const Slice& /*key*/, const Slice& left_operand,
const Slice& right_operand, std::string* new_value,
Logger* /*logger*/) const override;
bool PartialMergeMulti(const Slice& /*key*/,
const std::deque<Slice>& operand_list,
std::string* new_value,
Logger* /*logger*/) const override;
};
} // namespace ROCKSDB_NAMESPACE

@ -8,12 +8,9 @@
#include "rocksdb/merge_operator.h" #include "rocksdb/merge_operator.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "utilities/merge_operators.h" #include "utilities/merge_operators.h"
#include "utilities/merge_operators/put_operator.h"
namespace { // anonymous namespace namespace ROCKSDB_NAMESPACE {
using ROCKSDB_NAMESPACE::Logger;
using ROCKSDB_NAMESPACE::MergeOperator;
using ROCKSDB_NAMESPACE::Slice;
// A merge operator that mimics Put semantics // A merge operator that mimics Put semantics
// Since this merge-operator will not be used in production, // Since this merge-operator will not be used in production,
@ -23,64 +20,49 @@ using ROCKSDB_NAMESPACE::Slice;
// which would be simpler in this case). // which would be simpler in this case).
// //
// From the client-perspective, semantics are the same. // From the client-perspective, semantics are the same.
class PutOperator : public MergeOperator { bool PutOperator::FullMerge(const Slice& /*key*/,
public: const Slice* /*existing_value*/,
bool FullMerge(const Slice& /*key*/, const Slice* /*existing_value*/, const std::deque<std::string>& operand_sequence,
const std::deque<std::string>& operand_sequence, std::string* new_value, Logger* /*logger*/) const {
std::string* new_value, Logger* /*logger*/) const override { // Put basically only looks at the current/latest value
// Put basically only looks at the current/latest value assert(!operand_sequence.empty());
assert(!operand_sequence.empty()); assert(new_value != nullptr);
assert(new_value != nullptr); new_value->assign(operand_sequence.back());
new_value->assign(operand_sequence.back()); return true;
return true; }
}
bool PartialMerge(const Slice& /*key*/, const Slice& /*left_operand*/,
const Slice& right_operand, std::string* new_value,
Logger* /*logger*/) const override {
new_value->assign(right_operand.data(), right_operand.size());
return true;
}
using MergeOperator::PartialMergeMulti;
bool PartialMergeMulti(const Slice& /*key*/,
const std::deque<Slice>& operand_list,
std::string* new_value,
Logger* /*logger*/) const override {
new_value->assign(operand_list.back().data(), operand_list.back().size());
return true;
}
static const char* kClassName() { return "PutOperator"; }
static const char* kNickName() { return "put_v1"; }
const char* Name() const override { return kClassName(); }
const char* NickName() const override { return kNickName(); }
};
class PutOperatorV2 : public PutOperator {
bool FullMerge(const Slice& /*key*/, const Slice* /*existing_value*/,
const std::deque<std::string>& /*operand_sequence*/,
std::string* /*new_value*/,
Logger* /*logger*/) const override {
assert(false);
return false;
}
bool FullMergeV2(const MergeOperationInput& merge_in, bool PutOperator::PartialMerge(const Slice& /*key*/,
MergeOperationOutput* merge_out) const override { const Slice& /*left_operand*/,
// Put basically only looks at the current/latest value const Slice& right_operand,
assert(!merge_in.operand_list.empty()); std::string* new_value,
merge_out->existing_operand = merge_in.operand_list.back(); Logger* /*logger*/) const {
return true; new_value->assign(right_operand.data(), right_operand.size());
} return true;
}
static const char* kNickName() { return "put"; } bool PutOperator::PartialMergeMulti(const Slice& /*key*/,
const char* NickName() const override { return kNickName(); } const std::deque<Slice>& operand_list,
}; std::string* new_value,
Logger* /*logger*/) const {
new_value->assign(operand_list.back().data(), operand_list.back().size());
return true;
}
} // end of anonymous namespace bool PutOperatorV2::FullMerge(
const Slice& /*key*/, const Slice* /*existing_value*/,
const std::deque<std::string>& /*operand_sequence*/,
std::string* /*new_value*/, Logger* /*logger*/) const {
assert(false);
return false;
}
namespace ROCKSDB_NAMESPACE { bool PutOperatorV2::FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const {
// Put basically only looks at the current/latest value
assert(!merge_in.operand_list.empty());
merge_out->existing_operand = merge_in.operand_list.back();
return true;
}
std::shared_ptr<MergeOperator> MergeOperators::CreateDeprecatedPutOperator() { std::shared_ptr<MergeOperator> MergeOperators::CreateDeprecatedPutOperator() {
return std::make_shared<PutOperator>(); return std::make_shared<PutOperator>();

@ -0,0 +1,56 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// A merge operator that mimics Put semantics
// Since this merge-operator will not be used in production,
// it is implemented as a non-associative merge operator to illustrate the
// new interface and for testing purposes. (That is, we inherit from
// the MergeOperator class rather than the AssociativeMergeOperator
// which would be simpler in this case).
//
// From the client-perspective, semantics are the same.
#pragma once
#include "rocksdb/merge_operator.h"
namespace ROCKSDB_NAMESPACE {
class Logger;
class Slice;
class PutOperator : public MergeOperator {
public:
static const char* kClassName() { return "PutOperator"; }
static const char* kNickName() { return "put_v1"; }
const char* Name() const override { return kClassName(); }
const char* NickName() const override { return kNickName(); }
bool FullMerge(const Slice& /*key*/, const Slice* /*existing_value*/,
const std::deque<std::string>& operand_sequence,
std::string* new_value, Logger* /*logger*/) const override;
bool PartialMerge(const Slice& /*key*/, const Slice& left_operand,
const Slice& right_operand, std::string* new_value,
Logger* /*logger*/) const override;
using MergeOperator::PartialMergeMulti;
bool PartialMergeMulti(const Slice& /*key*/,
const std::deque<Slice>& operand_list,
std::string* new_value,
Logger* /*logger*/) const override;
};
class PutOperatorV2 : public PutOperator {
public:
static const char* kNickName() { return "put"; }
const char* NickName() const override { return kNickName(); }
bool FullMerge(const Slice& /*key*/, const Slice* /*existing_value*/,
const std::deque<std::string>& /*operand_sequence*/,
std::string* /*new_value*/, Logger* /*logger*/) const override;
bool FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const override;
};
} // namespace ROCKSDB_NAMESPACE

@ -3,6 +3,8 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "utilities/merge_operators/uint64add.h"
#include <memory> #include <memory>
#include "logging/logging.h" #include "logging/logging.h"
@ -12,61 +14,40 @@
#include "util/coding.h" #include "util/coding.h"
#include "utilities/merge_operators.h" #include "utilities/merge_operators.h"
namespace { // anonymous namespace namespace ROCKSDB_NAMESPACE { // anonymous namespace
using ROCKSDB_NAMESPACE::AssociativeMergeOperator;
using ROCKSDB_NAMESPACE::InfoLogLevel;
using ROCKSDB_NAMESPACE::Logger;
using ROCKSDB_NAMESPACE::Slice;
// A 'model' merge operator with uint64 addition semantics
// Implemented as an AssociativeMergeOperator for simplicity and example.
class UInt64AddOperator : public AssociativeMergeOperator {
public:
bool Merge(const Slice& /*key*/, const Slice* existing_value,
const Slice& value, std::string* new_value,
Logger* logger) const override {
uint64_t orig_value = 0;
if (existing_value) {
orig_value = DecodeInteger(*existing_value, logger);
}
uint64_t operand = DecodeInteger(value, logger);
assert(new_value);
new_value->clear();
ROCKSDB_NAMESPACE::PutFixed64(new_value, orig_value + operand);
return true; // Return true always since corruption will be treated as 0 bool UInt64AddOperator::Merge(const Slice& /*key*/, const Slice* existing_value,
const Slice& value, std::string* new_value,
Logger* logger) const {
uint64_t orig_value = 0;
if (existing_value) {
orig_value = DecodeInteger(*existing_value, logger);
} }
uint64_t operand = DecodeInteger(value, logger);
static const char* kClassName() { return "UInt64AddOperator"; } assert(new_value);
static const char* kNickName() { return "uint64add"; } new_value->clear();
const char* Name() const override { return kClassName(); } PutFixed64(new_value, orig_value + operand);
const char* NickName() const override { return kNickName(); }
private: return true; // Return true always since corruption will be treated as 0
// Takes the string and decodes it into a uint64_t }
// On error, prints a message and returns 0
uint64_t DecodeInteger(const Slice& value, Logger* logger) const {
uint64_t result = 0;
if (value.size() == sizeof(uint64_t)) {
result = ROCKSDB_NAMESPACE::DecodeFixed64(value.data());
} else if (logger != nullptr) {
// If value is corrupted, treat it as 0
ROCKS_LOG_ERROR(logger,
"uint64 value corruption, size: %" ROCKSDB_PRIszt
" > %" ROCKSDB_PRIszt,
value.size(), sizeof(uint64_t));
}
return result; uint64_t UInt64AddOperator::DecodeInteger(const Slice& value,
Logger* logger) const {
uint64_t result = 0;
if (value.size() == sizeof(uint64_t)) {
result = DecodeFixed64(value.data());
} else if (logger != nullptr) {
// If value is corrupted, treat it as 0
ROCKS_LOG_ERROR(logger,
"uint64 value corruption, size: %" ROCKSDB_PRIszt
" > %" ROCKSDB_PRIszt,
value.size(), sizeof(uint64_t));
} }
};
} // anonymous namespace return result;
}
namespace ROCKSDB_NAMESPACE {
std::shared_ptr<MergeOperator> MergeOperators::CreateUInt64AddOperator() { std::shared_ptr<MergeOperator> MergeOperators::CreateUInt64AddOperator() {
return std::make_shared<UInt64AddOperator>(); return std::make_shared<UInt64AddOperator>();

@ -0,0 +1,35 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// A 'model' merge operator with uint64 addition semantics
// Implemented as an AssociativeMergeOperator for simplicity and example.
#pragma once
#include "rocksdb/merge_operator.h"
#include "utilities/merge_operators.h"
namespace ROCKSDB_NAMESPACE {
class Logger;
class Slice;
class UInt64AddOperator : public AssociativeMergeOperator {
public:
static const char* kClassName() { return "UInt64AddOperator"; }
static const char* kNickName() { return "uint64add"; }
const char* Name() const override { return kClassName(); }
const char* NickName() const override { return kNickName(); }
bool Merge(const Slice& /*key*/, const Slice* existing_value,
const Slice& value, std::string* new_value,
Logger* logger) const override;
private:
// Takes the string and decodes it into a uint64_t
// On error, prints a message and returns 0
uint64_t DecodeInteger(const Slice& value, Logger* logger) const;
};
} // namespace ROCKSDB_NAMESPACE

@ -215,7 +215,7 @@ Status TablePropertiesCollectorFactory::CreateFromString(
""); "");
}); });
return LoadSharedObject<TablePropertiesCollectorFactory>(options, value, return LoadSharedObject<TablePropertiesCollectorFactory>(options, value,
nullptr, result); result);
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -15,8 +15,7 @@ namespace ROCKSDB_NAMESPACE {
Status WalFilter::CreateFromString(const ConfigOptions& config_options, Status WalFilter::CreateFromString(const ConfigOptions& config_options,
const std::string& value, const std::string& value,
WalFilter** filter) { WalFilter** filter) {
Status s = Status s = LoadStaticObject<WalFilter>(config_options, value, filter);
LoadStaticObject<WalFilter>(config_options, value, nullptr, filter);
return s; return s;
} }

Loading…
Cancel
Save