Make WalFilter, SstPartitionerFactory, FileChecksumGenFactory, and TableProperties Customizable (#8638)

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8638

Reviewed By: zhichao-cao

Differential Revision: D31024729

Pulled By: mrambacher

fbshipit-source-id: 954c04ccab0b8dee64050a27aadf78ed119106c0
main
mrambacher 3 years ago committed by Facebook GitHub Bot
parent b88109db19
commit 7fd68b7c39
  1. 1
      CMakeLists.txt
  2. 1
      HISTORY.md
  3. 2
      TARGETS
  4. 46
      db/compaction/sst_partitioner.cc
  5. 31
      db/db_table_properties_test.cc
  6. 7
      include/rocksdb/concurrent_task_limiter.h
  7. 1
      include/rocksdb/configurable.h
  8. 7
      include/rocksdb/file_checksum.h
  9. 14
      include/rocksdb/sst_partitioner.h
  10. 11
      include/rocksdb/table_properties.h
  11. 41
      include/rocksdb/utilities/table_properties_collectors.h
  12. 7
      include/rocksdb/wal_filter.h
  13. 12
      options/cf_options.cc
  14. 117
      options/customizable_test.cc
  15. 9
      options/db_options.cc
  16. 128
      options/options_test.cc
  17. 1
      src.mk
  18. 36
      util/file_checksum_helper.cc
  19. 3
      util/file_checksum_helper.h
  20. 109
      utilities/table_properties_collectors/compact_on_deletion_collector.cc
  21. 23
      utilities/wal_filter.cc

@ -903,6 +903,7 @@ set(SOURCES
utilities/transactions/write_unprepared_txn.cc utilities/transactions/write_unprepared_txn.cc
utilities/transactions/write_unprepared_txn_db.cc utilities/transactions/write_unprepared_txn_db.cc
utilities/ttl/db_ttl_impl.cc utilities/ttl/db_ttl_impl.cc
utilities/wal_filter.cc
utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index.cc
utilities/write_batch_with_index/write_batch_with_index_internal.cc) utilities/write_batch_with_index/write_batch_with_index_internal.cc)

@ -9,6 +9,7 @@
### Public API change ### Public API change
* Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. * Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method.
* Made SliceTransform extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. The Capped and Prefixed transform classes return a short name (no length); use GetId for the fully qualified name. * Made SliceTransform extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. The Capped and Prefixed transform classes return a short name (no length); use GetId for the fully qualified name.
* Made FileChecksumGenFactory, SstPartitionerFactory, TablePropertiesCollectorFactory, and WalFilter extend the Customizable class and added a CreateFromString method.
## 6.25.0 (2021-09-20) ## 6.25.0 (2021-09-20)
### Bug Fixes ### Bug Fixes

@ -435,6 +435,7 @@ cpp_library(
"utilities/transactions/write_unprepared_txn.cc", "utilities/transactions/write_unprepared_txn.cc",
"utilities/transactions/write_unprepared_txn_db.cc", "utilities/transactions/write_unprepared_txn_db.cc",
"utilities/ttl/db_ttl_impl.cc", "utilities/ttl/db_ttl_impl.cc",
"utilities/wal_filter.cc",
"utilities/write_batch_with_index/write_batch_with_index.cc", "utilities/write_batch_with_index/write_batch_with_index.cc",
"utilities/write_batch_with_index/write_batch_with_index_internal.cc", "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
], ],
@ -757,6 +758,7 @@ cpp_library(
"utilities/transactions/write_unprepared_txn.cc", "utilities/transactions/write_unprepared_txn.cc",
"utilities/transactions/write_unprepared_txn_db.cc", "utilities/transactions/write_unprepared_txn_db.cc",
"utilities/ttl/db_ttl_impl.cc", "utilities/ttl/db_ttl_impl.cc",
"utilities/wal_filter.cc",
"utilities/write_batch_with_index/write_batch_with_index.cc", "utilities/write_batch_with_index/write_batch_with_index.cc",
"utilities/write_batch_with_index/write_batch_with_index_internal.cc", "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
], ],

@ -8,7 +8,24 @@
#include <algorithm> #include <algorithm>
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
static std::unordered_map<std::string, OptionTypeInfo>
sst_fixed_prefix_type_info = {
#ifndef ROCKSDB_LITE
{"length",
{0, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
SstPartitionerFixedPrefixFactory::SstPartitionerFixedPrefixFactory(size_t len)
: len_(len) {
RegisterOptions("Length", &len_, &sst_fixed_prefix_type_info);
}
PartitionerResult SstPartitionerFixedPrefix::ShouldPartition( PartitionerResult SstPartitionerFixedPrefix::ShouldPartition(
const PartitionerRequest& request) { const PartitionerRequest& request) {
@ -41,4 +58,33 @@ std::shared_ptr<SstPartitionerFactory> NewSstPartitionerFixedPrefixFactory(
return std::make_shared<SstPartitionerFixedPrefixFactory>(prefix_len); return std::make_shared<SstPartitionerFixedPrefixFactory>(prefix_len);
} }
#ifndef ROCKSDB_LITE
namespace {
static int RegisterSstPartitionerFactories(ObjectLibrary& library,
const std::string& /*arg*/) {
library.Register<SstPartitionerFactory>(
SstPartitionerFixedPrefixFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<SstPartitionerFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new SstPartitionerFixedPrefixFactory(0));
return guard->get();
});
return 1;
}
} // namespace
#endif // ROCKSDB_LITE
Status SstPartitionerFactory::CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<SstPartitionerFactory>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterSstPartitionerFactories(*(ObjectLibrary::Default().get()), "");
});
#endif // ROCKSDB_LITE
return LoadSharedObject<SstPartitionerFactory>(options, value, nullptr,
result);
}
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -94,6 +94,37 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
VerifyTableProperties(db_, 10 + 11 + 12 + 13); VerifyTableProperties(db_, 10 + 11 + 12 + 13);
} }
TEST_F(DBTablePropertiesTest, CreateOnDeletionCollectorFactory) {
ConfigOptions options;
options.ignore_unsupported_options = false;
std::shared_ptr<TablePropertiesCollectorFactory> factory;
std::string id = CompactOnDeletionCollectorFactory::kClassName();
ASSERT_OK(
TablePropertiesCollectorFactory::CreateFromString(options, id, &factory));
auto del_factory = factory->CheckedCast<CompactOnDeletionCollectorFactory>();
ASSERT_NE(del_factory, nullptr);
ASSERT_EQ(0U, del_factory->GetWindowSize());
ASSERT_EQ(0U, del_factory->GetDeletionTrigger());
ASSERT_EQ(0.0, del_factory->GetDeletionRatio());
ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString(
options, "window_size=100; deletion_trigger=90; id=" + id, &factory));
del_factory = factory->CheckedCast<CompactOnDeletionCollectorFactory>();
ASSERT_NE(del_factory, nullptr);
ASSERT_EQ(100U, del_factory->GetWindowSize());
ASSERT_EQ(90U, del_factory->GetDeletionTrigger());
ASSERT_EQ(0.0, del_factory->GetDeletionRatio());
ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString(
options,
"window_size=100; deletion_trigger=90; deletion_ratio=0.5; id=" + id,
&factory));
del_factory = factory->CheckedCast<CompactOnDeletionCollectorFactory>();
ASSERT_NE(del_factory, nullptr);
ASSERT_EQ(100U, del_factory->GetWindowSize());
ASSERT_EQ(90U, del_factory->GetDeletionTrigger());
ASSERT_EQ(0.5, del_factory->GetDeletionRatio());
}
TablePropertiesCollection TablePropertiesCollection
DBTablePropertiesTest::TestGetPropertiesOfTablesInRange( DBTablePropertiesTest::TestGetPropertiesOfTablesInRange(
std::vector<Range> ranges, std::size_t* num_properties, std::vector<Range> ranges, std::size_t* num_properties,

@ -9,8 +9,11 @@
#pragma once #pragma once
#include "rocksdb/env.h" #include <stdint.h>
#include "rocksdb/statistics.h"
#include <string>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -261,7 +261,6 @@ class Configurable {
virtual Status ValidateOptions(const DBOptions& db_opts, virtual Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const; const ColumnFamilyOptions& cf_opts) const;
// Splits the input opt_value into the ID field and the remaining options. // Splits the input opt_value into the ID field and the remaining options.
// The input opt_value can be in the form of "name" or "name=value // The input opt_value can be in the form of "name" or "name=value
// [;name=value]". The first form uses the "name" as an id with no options The // [;name=value]". The first form uses the "name" as an id with no options The

@ -14,6 +14,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "rocksdb/customizable.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -63,9 +64,13 @@ class FileChecksumGenerator {
}; };
// Create the FileChecksumGenerator object for each SST file. // Create the FileChecksumGenerator object for each SST file.
class FileChecksumGenFactory { class FileChecksumGenFactory : public Customizable {
public: public:
virtual ~FileChecksumGenFactory() {} virtual ~FileChecksumGenFactory() {}
static const char* Type() { return "FileChecksumGenFactory"; }
static Status CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<FileChecksumGenFactory>* result);
// Create a new FileChecksumGenerator. // Create a new FileChecksumGenerator.
virtual std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator( virtual std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(

@ -9,6 +9,7 @@
#include <memory> #include <memory>
#include <string> #include <string>
#include "rocksdb/customizable.h"
#include "rocksdb/rocksdb_namespace.h" #include "rocksdb/rocksdb_namespace.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
@ -77,9 +78,13 @@ class SstPartitioner {
}; };
}; };
class SstPartitionerFactory { class SstPartitionerFactory : public Customizable {
public: public:
virtual ~SstPartitionerFactory() {} virtual ~SstPartitionerFactory() {}
static const char* Type() { return "SstPartitionerFactory"; }
static Status CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<SstPartitionerFactory>* result);
virtual std::unique_ptr<SstPartitioner> CreatePartitioner( virtual std::unique_ptr<SstPartitioner> CreatePartitioner(
const SstPartitioner::Context& context) const = 0; const SstPartitioner::Context& context) const = 0;
@ -114,13 +119,12 @@ class SstPartitionerFixedPrefix : public SstPartitioner {
*/ */
class SstPartitionerFixedPrefixFactory : public SstPartitionerFactory { class SstPartitionerFixedPrefixFactory : public SstPartitionerFactory {
public: public:
explicit SstPartitionerFixedPrefixFactory(size_t len) : len_(len) {} explicit SstPartitionerFixedPrefixFactory(size_t len);
virtual ~SstPartitionerFixedPrefixFactory() {} virtual ~SstPartitionerFixedPrefixFactory() {}
const char* Name() const override { static const char* kClassName() { return "SstPartitionerFixedPrefixFactory"; }
return "SstPartitionerFixedPrefixFactory"; const char* Name() const override { return kClassName(); }
}
std::unique_ptr<SstPartitioner> CreatePartitioner( std::unique_ptr<SstPartitioner> CreatePartitioner(
const SstPartitioner::Context& /* context */) const override; const SstPartitioner::Context& /* context */) const override;

@ -5,8 +5,12 @@
#pragma once #pragma once
#include <stdint.h> #include <stdint.h>
#include <map> #include <map>
#include <memory>
#include <string> #include <string>
#include "rocksdb/customizable.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/types.h" #include "rocksdb/types.h"
@ -129,7 +133,7 @@ class TablePropertiesCollector {
// Constructs TablePropertiesCollector. Internals create a new // Constructs TablePropertiesCollector. Internals create a new
// TablePropertiesCollector for each new table // TablePropertiesCollector for each new table
class TablePropertiesCollectorFactory { class TablePropertiesCollectorFactory : public Customizable {
public: public:
struct Context { struct Context {
uint32_t column_family_id; uint32_t column_family_id;
@ -137,6 +141,11 @@ class TablePropertiesCollectorFactory {
}; };
virtual ~TablePropertiesCollectorFactory() {} virtual ~TablePropertiesCollectorFactory() {}
static const char* Type() { return "TablePropertiesCollectorFactory"; }
static Status CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<TablePropertiesCollectorFactory>* result);
// has to be thread-safe // has to be thread-safe
virtual TablePropertiesCollector* CreateTablePropertiesCollector( virtual TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context context) = 0; TablePropertiesCollectorFactory::Context context) = 0;

@ -19,6 +19,19 @@ namespace ROCKSDB_NAMESPACE {
class CompactOnDeletionCollectorFactory class CompactOnDeletionCollectorFactory
: public TablePropertiesCollectorFactory { : public TablePropertiesCollectorFactory {
public: public:
// A factory of a table property collector that marks a SST
// file as need-compaction when it observe at least "D" deletion
// entries in any "N" consecutive entries, or the ratio of tombstone
// entries >= deletion_ratio.
//
// @param sliding_window_size "N"
// @param deletion_trigger "D"
// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
// based on deletion ratio.
CompactOnDeletionCollectorFactory(size_t sliding_window_size,
size_t deletion_trigger,
double deletion_ratio);
~CompactOnDeletionCollectorFactory() {} ~CompactOnDeletionCollectorFactory() {}
TablePropertiesCollector* CreateTablePropertiesCollector( TablePropertiesCollector* CreateTablePropertiesCollector(
@ -29,12 +42,14 @@ class CompactOnDeletionCollectorFactory
void SetWindowSize(size_t sliding_window_size) { void SetWindowSize(size_t sliding_window_size) {
sliding_window_size_.store(sliding_window_size); sliding_window_size_.store(sliding_window_size);
} }
size_t GetWindowSize() const { return sliding_window_size_.load(); }
// Change the value of deletion_trigger "D" // Change the value of deletion_trigger "D"
void SetDeletionTrigger(size_t deletion_trigger) { void SetDeletionTrigger(size_t deletion_trigger) {
deletion_trigger_.store(deletion_trigger); deletion_trigger_.store(deletion_trigger);
} }
size_t GetDeletionTrigger() const { return deletion_trigger_.load(); }
// Change deletion ratio. // Change deletion ratio.
// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
// based on deletion ratio. // based on deletion ratio.
@ -42,33 +57,13 @@ class CompactOnDeletionCollectorFactory
deletion_ratio_.store(deletion_ratio); deletion_ratio_.store(deletion_ratio);
} }
const char* Name() const override { double GetDeletionRatio() const { return deletion_ratio_.load(); }
return "CompactOnDeletionCollector"; static const char* kClassName() { return "CompactOnDeletionCollector"; }
} const char* Name() const override { return kClassName(); }
std::string ToString() const override; std::string ToString() const override;
private: private:
friend std::shared_ptr<CompactOnDeletionCollectorFactory>
NewCompactOnDeletionCollectorFactory(size_t sliding_window_size,
size_t deletion_trigger,
double deletion_ratio);
// A factory of a table property collector that marks a SST
// file as need-compaction when it observe at least "D" deletion
// entries in any "N" consecutive entries, or the ratio of tombstone
// entries >= deletion_ratio.
//
// @param sliding_window_size "N"
// @param deletion_trigger "D"
// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
// based on deletion ratio.
CompactOnDeletionCollectorFactory(size_t sliding_window_size,
size_t deletion_trigger,
double deletion_ratio)
: sliding_window_size_(sliding_window_size),
deletion_trigger_(deletion_trigger),
deletion_ratio_(deletion_ratio) {}
std::atomic<size_t> sliding_window_size_; std::atomic<size_t> sliding_window_size_;
std::atomic<size_t> deletion_trigger_; std::atomic<size_t> deletion_trigger_;
std::atomic<double> deletion_ratio_; std::atomic<double> deletion_ratio_;

@ -8,17 +8,22 @@
#include <map> #include <map>
#include <string> #include <string>
#include "rocksdb/customizable.h"
#include "rocksdb/rocksdb_namespace.h" #include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class WriteBatch; class WriteBatch;
struct ConfigOptions;
// WALFilter allows an application to inspect write-ahead-log (WAL) // WALFilter allows an application to inspect write-ahead-log (WAL)
// records or modify their processing on recovery. // records or modify their processing on recovery.
// Please see the details below. // Please see the details below.
class WalFilter { class WalFilter : public Customizable {
public: public:
static const char* Type() { return "WalFilter"; }
static Status CreateFromString(const ConfigOptions& options,
const std::string& value, WalFilter** result);
enum class WalProcessingOption { enum class WalProcessingOption {
// Continue processing as usual // Continue processing as usual
kContinueProcessing = 0, kContinueProcessing = 0,

@ -673,6 +673,14 @@ static std::unordered_map<std::string, OptionTypeInfo>
return Status::NotFound("Mismatched table option: ", name); return Status::NotFound("Mismatched table option: ", name);
} }
}}}, }}},
{"table_properties_collectors",
OptionTypeInfo::Vector<
std::shared_ptr<TablePropertiesCollectorFactory>>(
offset_of(
&ImmutableCFOptions::table_properties_collector_factories),
OptionVerificationType::kByName, OptionTypeFlags::kNone,
OptionTypeInfo::AsCustomSharedPtr<TablePropertiesCollectorFactory>(
0, OptionVerificationType::kByName, OptionTypeFlags::kNone))},
{"compaction_filter", {"compaction_filter",
OptionTypeInfo::AsCustomRawPtr<const CompactionFilter>( OptionTypeInfo::AsCustomRawPtr<const CompactionFilter>(
offset_of(&ImmutableCFOptions::compaction_filter), offset_of(&ImmutableCFOptions::compaction_filter),
@ -694,6 +702,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offset_of(&ImmutableCFOptions::compaction_pri), {offset_of(&ImmutableCFOptions::compaction_pri),
OptionType::kCompactionPri, OptionVerificationType::kNormal, OptionType::kCompactionPri, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}}, OptionTypeFlags::kNone}},
{"sst_partitioner_factory",
OptionTypeInfo::AsCustomSharedPtr<SstPartitionerFactory>(
offset_of(&ImmutableCFOptions::sst_partitioner_factory),
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
}; };
const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions"; const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions";

@ -20,9 +20,11 @@
#include "port/stack_trace.h" #include "port/stack_trace.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/env_encryption.h" #include "rocksdb/env_encryption.h"
#include "rocksdb/file_checksum.h"
#include "rocksdb/flush_block_policy.h" #include "rocksdb/flush_block_policy.h"
#include "rocksdb/secondary_cache.h" #include "rocksdb/secondary_cache.h"
#include "rocksdb/slice_transform.h" #include "rocksdb/slice_transform.h"
#include "rocksdb/sst_partitioner.h"
#include "rocksdb/statistics.h" #include "rocksdb/statistics.h"
#include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/object_registry.h"
@ -32,6 +34,7 @@
#include "test_util/mock_time_env.h" #include "test_util/mock_time_env.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
#include "test_util/testutil.h" #include "test_util/testutil.h"
#include "util/file_checksum_helper.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h" #include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h"
@ -1297,6 +1300,41 @@ class MockCipher : public BlockCipher {
Status Decrypt(char* data) override { return Encrypt(data); } Status Decrypt(char* data) override { return Encrypt(data); }
}; };
#endif // ROCKSDB_LITE
class MockTablePropertiesCollectorFactory
: public TablePropertiesCollectorFactory {
private:
public:
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context /*context*/) override {
return nullptr;
}
static const char* kClassName() { return "Mock"; }
const char* Name() const override { return kClassName(); }
};
class MockSstPartitionerFactory : public SstPartitionerFactory {
public:
static const char* kClassName() { return "Mock"; }
const char* Name() const override { return kClassName(); }
std::unique_ptr<SstPartitioner> CreatePartitioner(
const SstPartitioner::Context& /* context */) const override {
return nullptr;
}
};
class MockFileChecksumGenFactory : public FileChecksumGenFactory {
public:
static const char* kClassName() { return "Mock"; }
const char* Name() const override { return kClassName(); }
std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
const FileChecksumGenContext& /*context*/) override {
return nullptr;
}
};
#ifndef ROCKSDB_LITE
static int RegisterLocalObjects(ObjectLibrary& library, static int RegisterLocalObjects(ObjectLibrary& library,
const std::string& /*arg*/) { const std::string& /*arg*/) {
size_t num_types; size_t num_types;
@ -1367,6 +1405,33 @@ static int RegisterLocalObjects(ObjectLibrary& library,
guard->reset(new TestSecondaryCache()); guard->reset(new TestSecondaryCache());
return guard->get(); return guard->get();
}); });
library.Register<SstPartitionerFactory>(
MockSstPartitionerFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<SstPartitionerFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new MockSstPartitionerFactory());
return guard->get();
});
library.Register<FileChecksumGenFactory>(
MockFileChecksumGenFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<FileChecksumGenFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new MockFileChecksumGenFactory());
return guard->get();
});
library.Register<TablePropertiesCollectorFactory>(
MockTablePropertiesCollectorFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<TablePropertiesCollectorFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new MockTablePropertiesCollectorFactory());
return guard->get();
});
return static_cast<int>(library.GetFactoryCount(&num_types)); return static_cast<int>(library.GetFactoryCount(&num_types));
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
@ -1443,6 +1508,58 @@ TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) {
} }
} }
#ifndef ROCKSDB_LITE
TEST_F(LoadCustomizableTest, LoadSstPartitionerFactoryTest) {
std::shared_ptr<SstPartitionerFactory> factory;
ASSERT_NOK(SstPartitionerFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_OK(SstPartitionerFactory::CreateFromString(
config_options_, SstPartitionerFixedPrefixFactory::kClassName(),
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), SstPartitionerFixedPrefixFactory::kClassName());
if (RegisterTests("Test")) {
ASSERT_OK(SstPartitionerFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), "Mock");
}
}
#endif // ROCKSDB_LITE
TEST_F(LoadCustomizableTest, LoadChecksumGenFactoryTest) {
std::shared_ptr<FileChecksumGenFactory> factory;
ASSERT_NOK(FileChecksumGenFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_OK(FileChecksumGenFactory::CreateFromString(
config_options_, FileChecksumGenCrc32cFactory::kClassName(), &factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), FileChecksumGenCrc32cFactory::kClassName());
if (RegisterTests("Test")) {
ASSERT_OK(FileChecksumGenFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), "Mock");
}
}
TEST_F(LoadCustomizableTest, LoadTablePropertiesCollectorFactoryTest) {
std::shared_ptr<TablePropertiesCollectorFactory> factory;
ASSERT_NOK(TablePropertiesCollectorFactory::CreateFromString(
config_options_, MockTablePropertiesCollectorFactory::kClassName(),
&factory));
if (RegisterTests("Test")) {
ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString(
config_options_, MockTablePropertiesCollectorFactory::kClassName(),
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(),
MockTablePropertiesCollectorFactory::kClassName());
}
}
TEST_F(LoadCustomizableTest, LoadComparatorTest) { TEST_F(LoadCustomizableTest, LoadComparatorTest) {
const Comparator* bytewise = BytewiseComparator(); const Comparator* bytewise = BytewiseComparator();
const Comparator* reverse = ReverseBytewiseComparator(); const Comparator* reverse = ReverseBytewiseComparator();

@ -171,6 +171,11 @@ static std::unordered_map<std::string, OptionTypeInfo>
{"allow_2pc", {"allow_2pc",
{offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"wal_filter",
OptionTypeInfo::AsCustomRawPtr<WalFilter>(
offsetof(struct ImmutableDBOptions, wal_filter),
OptionVerificationType::kByName,
(OptionTypeFlags::kAllowNull | OptionTypeFlags::kCompareNever))},
{"create_if_missing", {"create_if_missing",
{offsetof(struct ImmutableDBOptions, create_if_missing), {offsetof(struct ImmutableDBOptions, create_if_missing),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
@ -444,6 +449,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct ImmutableDBOptions, allow_data_in_errors), {offsetof(struct ImmutableDBOptions, allow_data_in_errors),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}}, OptionTypeFlags::kNone}},
{"file_checksum_gen_factory",
OptionTypeInfo::AsCustomSharedPtr<FileChecksumGenFactory>(
offsetof(struct ImmutableDBOptions, file_checksum_gen_factory),
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
{"statistics", {"statistics",
OptionTypeInfo::AsCustomSharedPtr<Statistics>( OptionTypeInfo::AsCustomSharedPtr<Statistics>(
// Statistics should not be compared and can be null // Statistics should not be compared and can be null

@ -19,6 +19,7 @@
#include "port/port.h" #include "port/port.h"
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/file_checksum.h"
#include "rocksdb/memtablerep.h" #include "rocksdb/memtablerep.h"
#include "rocksdb/utilities/leveldb_options.h" #include "rocksdb/utilities/leveldb_options.h"
#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/object_registry.h"
@ -1960,6 +1961,133 @@ TEST_F(OptionsTest, OnlyMutableCFOptions) {
opt_str, &cf_opts)); opt_str, &cf_opts));
delete cf_opts.compaction_filter; delete cf_opts.compaction_filter;
} }
TEST_F(OptionsTest, SstPartitionerTest) {
ConfigOptions cfg_opts;
ColumnFamilyOptions cf_opts, new_opt;
std::string opts_str, mismatch;
ASSERT_OK(SstPartitionerFactory::CreateFromString(
cfg_opts, SstPartitionerFixedPrefixFactory::kClassName(),
&cf_opts.sst_partitioner_factory));
ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr);
ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(),
SstPartitionerFixedPrefixFactory::kClassName());
ASSERT_NOK(GetColumnFamilyOptionsFromString(
cfg_opts, ColumnFamilyOptions(),
std::string("sst_partitioner_factory={id=") +
SstPartitionerFixedPrefixFactory::kClassName() + "; unknown=10;}",
&cf_opts));
ASSERT_OK(GetColumnFamilyOptionsFromString(
cfg_opts, ColumnFamilyOptions(),
std::string("sst_partitioner_factory={id=") +
SstPartitionerFixedPrefixFactory::kClassName() + "; length=10;}",
&cf_opts));
ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr);
ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(),
SstPartitionerFixedPrefixFactory::kClassName());
ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, cf_opts, &opts_str));
ASSERT_OK(
GetColumnFamilyOptionsFromString(cfg_opts, cf_opts, opts_str, &new_opt));
ASSERT_NE(new_opt.sst_partitioner_factory, nullptr);
ASSERT_STREQ(new_opt.sst_partitioner_factory->Name(),
SstPartitionerFixedPrefixFactory::kClassName());
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, cf_opts, new_opt));
ASSERT_TRUE(cf_opts.sst_partitioner_factory->AreEquivalent(
cfg_opts, new_opt.sst_partitioner_factory.get(), &mismatch));
}
TEST_F(OptionsTest, FileChecksumGenFactoryTest) {
ConfigOptions cfg_opts;
DBOptions db_opts, new_opt;
std::string opts_str, mismatch;
auto factory = GetFileChecksumGenCrc32cFactory();
cfg_opts.ignore_unsupported_options = false;
ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str));
ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt));
ASSERT_NE(factory, nullptr);
ASSERT_OK(FileChecksumGenFactory::CreateFromString(
cfg_opts, factory->Name(), &db_opts.file_checksum_gen_factory));
ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr);
ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name());
ASSERT_NOK(GetDBOptionsFromString(
cfg_opts, DBOptions(), "file_checksum_gen_factory=unknown", &db_opts));
ASSERT_OK(GetDBOptionsFromString(
cfg_opts, DBOptions(),
std::string("file_checksum_gen_factory=") + factory->Name(), &db_opts));
ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr);
ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name());
ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str));
ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt));
ASSERT_NE(new_opt.file_checksum_gen_factory, nullptr);
ASSERT_STREQ(new_opt.file_checksum_gen_factory->Name(), factory->Name());
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(cfg_opts, db_opts, new_opt));
ASSERT_TRUE(factory->AreEquivalent(
cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch));
ASSERT_TRUE(db_opts.file_checksum_gen_factory->AreEquivalent(
cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch));
}
class TestTablePropertiesCollectorFactory
: public TablePropertiesCollectorFactory {
private:
std::string id_;
public:
explicit TestTablePropertiesCollectorFactory(const std::string& id)
: id_(id) {}
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context /*context*/) override {
return nullptr;
}
static const char* kClassName() { return "TestCollector"; }
const char* Name() const override { return kClassName(); }
std::string GetId() const override {
return std::string(kClassName()) + ":" + id_;
}
};
TEST_F(OptionsTest, OptionTablePropertiesTest) {
ConfigOptions cfg_opts;
ColumnFamilyOptions orig, copy;
orig.table_properties_collector_factories.push_back(
std::make_shared<TestTablePropertiesCollectorFactory>("1"));
orig.table_properties_collector_factories.push_back(
std::make_shared<TestTablePropertiesCollectorFactory>("2"));
// Push two TablePropertiesCollectorFactories then create a new
// ColumnFamilyOptions based on those settings. The copy should
// have no properties but still match the original
std::string opts_str;
ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, orig, &opts_str));
ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, &copy));
ASSERT_EQ(copy.table_properties_collector_factories.size(), 0);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy));
// Now register a TablePropertiesCollectorFactory
// Repeat the experiment. The copy should have the same
// properties as the original
cfg_opts.registry->AddLibrary("collector")
->Register<TablePropertiesCollectorFactory>(
std::string(TestTablePropertiesCollectorFactory::kClassName()) +
":.*",
[](const std::string& name,
std::unique_ptr<TablePropertiesCollectorFactory>* guard,
std::string* /* errmsg */) {
std::string id = name.substr(
strlen(TestTablePropertiesCollectorFactory::kClassName()) + 1);
guard->reset(new TestTablePropertiesCollectorFactory(id));
return guard->get();
});
ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, &copy));
ASSERT_EQ(copy.table_properties_collector_factories.size(), 2);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy));
}
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
TEST_F(OptionsTest, ConvertOptionsTest) { TEST_F(OptionsTest, ConvertOptionsTest) {

@ -285,6 +285,7 @@ LIB_SOURCES = \
utilities/transactions/write_unprepared_txn.cc \ utilities/transactions/write_unprepared_txn.cc \
utilities/transactions/write_unprepared_txn_db.cc \ utilities/transactions/write_unprepared_txn_db.cc \
utilities/ttl/db_ttl_impl.cc \ utilities/ttl/db_ttl_impl.cc \
utilities/wal_filter.cc \
utilities/write_batch_with_index/write_batch_with_index.cc \ utilities/write_batch_with_index/write_batch_with_index.cc \
utilities/write_batch_with_index/write_batch_with_index_internal.cc \ utilities/write_batch_with_index/write_batch_with_index_internal.cc \

@ -15,6 +15,7 @@
#include "db/version_edit.h" #include "db/version_edit.h"
#include "db/version_edit_handler.h" #include "db/version_edit_handler.h"
#include "file/sequence_file_reader.h" #include "file/sequence_file_reader.h"
#include "rocksdb/utilities/customizable_util.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -133,4 +134,39 @@ Status GetFileChecksumsFromManifest(Env* src_env, const std::string& abs_path,
return retriever.status(); return retriever.status();
} }
#ifndef ROCKSDB_LITE
namespace {
static int RegisterFileChecksumGenFactories(ObjectLibrary& library,
const std::string& /*arg*/) {
library.Register<FileChecksumGenFactory>(
FileChecksumGenCrc32cFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<FileChecksumGenFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new FileChecksumGenCrc32cFactory());
return guard->get();
});
return 1;
}
} // namespace
#endif // !ROCKSDB_LITE
Status FileChecksumGenFactory::CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<FileChecksumGenFactory>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterFileChecksumGenFactories(*(ObjectLibrary::Default().get()), "");
});
#endif // ROCKSDB_LITE
if (value == FileChecksumGenCrc32cFactory::kClassName()) {
*result = GetFileChecksumGenCrc32cFactory();
return Status::OK();
} else {
Status s = LoadSharedObject<FileChecksumGenFactory>(options, value, nullptr,
result);
return s;
}
}
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -58,7 +58,8 @@ class FileChecksumGenCrc32cFactory : public FileChecksumGenFactory {
} }
} }
const char* Name() const override { return "FileChecksumGenCrc32cFactory"; } static const char* kClassName() { return "FileChecksumGenCrc32cFactory"; }
const char* Name() const override { return kClassName(); }
}; };
// The default implementaion of FileChecksumList // The default implementaion of FileChecksumList

@ -3,14 +3,19 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include "utilities/table_properties_collectors/compact_on_deletion_collector.h" #include "utilities/table_properties_collectors/compact_on_deletion_collector.h"
#include <memory> #include <memory>
#include <sstream> #include <sstream>
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "rocksdb/utilities/table_properties_collectors.h" #include "rocksdb/utilities/table_properties_collectors.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
CompactOnDeletionCollector::CompactOnDeletionCollector( CompactOnDeletionCollector::CompactOnDeletionCollector(
size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio) size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio)
@ -93,6 +98,74 @@ Status CompactOnDeletionCollector::Finish(
finished_ = true; finished_ = true;
return Status::OK(); return Status::OK();
} }
static std::unordered_map<std::string, OptionTypeInfo>
on_deletion_collector_type_info = {
#ifndef ROCKSDB_LITE
{"window_size",
{0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
[](const ConfigOptions&, const std::string&, const std::string& value,
void* addr) {
auto* factory =
static_cast<CompactOnDeletionCollectorFactory*>(addr);
factory->SetWindowSize(ParseSizeT(value));
return Status::OK();
},
[](const ConfigOptions&, const std::string&, const void* addr,
std::string* value) {
const auto* factory =
static_cast<const CompactOnDeletionCollectorFactory*>(addr);
*value = ToString(factory->GetWindowSize());
return Status::OK();
},
nullptr}},
{"deletion_trigger",
{0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
[](const ConfigOptions&, const std::string&, const std::string& value,
void* addr) {
auto* factory =
static_cast<CompactOnDeletionCollectorFactory*>(addr);
factory->SetDeletionTrigger(ParseSizeT(value));
return Status::OK();
},
[](const ConfigOptions&, const std::string&, const void* addr,
std::string* value) {
const auto* factory =
static_cast<const CompactOnDeletionCollectorFactory*>(addr);
*value = ToString(factory->GetDeletionTrigger());
return Status::OK();
},
nullptr}},
{"deletion_ratio",
{0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
[](const ConfigOptions&, const std::string&, const std::string& value,
void* addr) {
auto* factory =
static_cast<CompactOnDeletionCollectorFactory*>(addr);
factory->SetDeletionRatio(ParseDouble(value));
return Status::OK();
},
[](const ConfigOptions&, const std::string&, const void* addr,
std::string* value) {
const auto* factory =
static_cast<const CompactOnDeletionCollectorFactory*>(addr);
*value = ToString(factory->GetDeletionRatio());
return Status::OK();
},
nullptr}},
#endif // ROCKSDB_LITE
};
CompactOnDeletionCollectorFactory::CompactOnDeletionCollectorFactory(
size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio)
: sliding_window_size_(sliding_window_size),
deletion_trigger_(deletion_trigger),
deletion_ratio_(deletion_ratio) {
RegisterOptions("", this, &on_deletion_collector_type_info);
}
TablePropertiesCollector* TablePropertiesCollector*
CompactOnDeletionCollectorFactory::CreateTablePropertiesCollector( CompactOnDeletionCollectorFactory::CreateTablePropertiesCollector(
@ -118,5 +191,37 @@ NewCompactOnDeletionCollectorFactory(size_t sliding_window_size,
new CompactOnDeletionCollectorFactory(sliding_window_size, new CompactOnDeletionCollectorFactory(sliding_window_size,
deletion_trigger, deletion_ratio)); deletion_trigger, deletion_ratio));
} }
} // namespace ROCKSDB_NAMESPACE namespace {
static int RegisterTablePropertiesCollectorFactories(
ObjectLibrary& library, const std::string& /*arg*/) {
library.Register<TablePropertiesCollectorFactory>(
CompactOnDeletionCollectorFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<TablePropertiesCollectorFactory>* guard,
std::string* /* errmsg */) {
// By default, create a CompactionOnDeletionCollector that is disabled.
// Users will need to provide configuration parameters or call the
// corresponding Setter to enable the factory.
guard->reset(new CompactOnDeletionCollectorFactory(0, 0, 0));
return guard->get();
});
return 1;
}
} // namespace
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
Status TablePropertiesCollectorFactory::CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<TablePropertiesCollectorFactory>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterTablePropertiesCollectorFactories(*(ObjectLibrary::Default().get()),
"");
});
#endif // ROCKSDB_LITE
return LoadSharedObject<TablePropertiesCollectorFactory>(options, value,
nullptr, result);
}
} // namespace ROCKSDB_NAMESPACE

@ -0,0 +1,23 @@
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/wal_filter.h"
#include <memory>
#include "rocksdb/convenience.h"
#include "rocksdb/options.h"
#include "rocksdb/utilities/customizable_util.h"
namespace ROCKSDB_NAMESPACE {
Status WalFilter::CreateFromString(const ConfigOptions& config_options,
const std::string& value,
WalFilter** filter) {
Status s =
LoadStaticObject<WalFilter>(config_options, value, nullptr, filter);
return s;
}
} // namespace ROCKSDB_NAMESPACE
Loading…
Cancel
Save