From 7fd68b7c39c76b5741533ee2a4d7024f3b513f2d Mon Sep 17 00:00:00 2001 From: mrambacher Date: Tue, 28 Sep 2021 05:30:32 -0700 Subject: [PATCH] Make WalFilter, SstPartitionerFactory, FileChecksumGenFactory, and TableProperties Customizable (#8638) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8638 Reviewed By: zhichao-cao Differential Revision: D31024729 Pulled By: mrambacher fbshipit-source-id: 954c04ccab0b8dee64050a27aadf78ed119106c0 --- CMakeLists.txt | 1 + HISTORY.md | 1 + TARGETS | 2 + db/compaction/sst_partitioner.cc | 46 +++++++ db/db_table_properties_test.cc | 31 +++++ include/rocksdb/concurrent_task_limiter.h | 7 +- include/rocksdb/configurable.h | 1 - include/rocksdb/file_checksum.h | 7 +- include/rocksdb/sst_partitioner.h | 14 +- include/rocksdb/table_properties.h | 11 +- .../utilities/table_properties_collectors.h | 41 +++--- include/rocksdb/wal_filter.h | 7 +- options/cf_options.cc | 12 ++ options/customizable_test.cc | 117 ++++++++++++++++ options/db_options.cc | 9 ++ options/options_test.cc | 128 ++++++++++++++++++ src.mk | 1 + util/file_checksum_helper.cc | 36 +++++ util/file_checksum_helper.h | 3 +- .../compact_on_deletion_collector.cc | 109 ++++++++++++++- utilities/wal_filter.cc | 23 ++++ 21 files changed, 570 insertions(+), 37 deletions(-) create mode 100644 utilities/wal_filter.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d9b4b64a..50b29096c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -903,6 +903,7 @@ set(SOURCES utilities/transactions/write_unprepared_txn.cc utilities/transactions/write_unprepared_txn_db.cc utilities/ttl/db_ttl_impl.cc + utilities/wal_filter.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc) diff --git a/HISTORY.md b/HISTORY.md index 8d373ceaa..7e20a0105 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -9,6 +9,7 @@ ### Public API change * Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. * Made SliceTransform extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. The Capped and Prefixed transform classes return a short name (no length); use GetId for the fully qualified name. +* Made FileChecksumGenFactory, SstPartitionerFactory, TablePropertiesCollectorFactory, and WalFilter extend the Customizable class and added a CreateFromString method. ## 6.25.0 (2021-09-20) ### Bug Fixes diff --git a/TARGETS b/TARGETS index 76266e74a..cb52c6052 100644 --- a/TARGETS +++ b/TARGETS @@ -435,6 +435,7 @@ cpp_library( "utilities/transactions/write_unprepared_txn.cc", "utilities/transactions/write_unprepared_txn_db.cc", "utilities/ttl/db_ttl_impl.cc", + "utilities/wal_filter.cc", "utilities/write_batch_with_index/write_batch_with_index.cc", "utilities/write_batch_with_index/write_batch_with_index_internal.cc", ], @@ -757,6 +758,7 @@ cpp_library( "utilities/transactions/write_unprepared_txn.cc", "utilities/transactions/write_unprepared_txn_db.cc", "utilities/ttl/db_ttl_impl.cc", + "utilities/wal_filter.cc", "utilities/write_batch_with_index/write_batch_with_index.cc", "utilities/write_batch_with_index/write_batch_with_index_internal.cc", ], diff --git a/db/compaction/sst_partitioner.cc b/db/compaction/sst_partitioner.cc index 1faa25707..36dcf80c5 100644 --- a/db/compaction/sst_partitioner.cc +++ b/db/compaction/sst_partitioner.cc @@ -8,7 +8,24 @@ #include +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" + namespace ROCKSDB_NAMESPACE { +static std::unordered_map + sst_fixed_prefix_type_info = { +#ifndef ROCKSDB_LITE + {"length", + {0, OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; + +SstPartitionerFixedPrefixFactory::SstPartitionerFixedPrefixFactory(size_t len) + : len_(len) { + RegisterOptions("Length", &len_, &sst_fixed_prefix_type_info); +} PartitionerResult SstPartitionerFixedPrefix::ShouldPartition( const PartitionerRequest& request) { @@ -41,4 +58,33 @@ std::shared_ptr NewSstPartitionerFixedPrefixFactory( return std::make_shared(prefix_len); } +#ifndef ROCKSDB_LITE +namespace { +static int RegisterSstPartitionerFactories(ObjectLibrary& library, + const std::string& /*arg*/) { + library.Register( + SstPartitionerFixedPrefixFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new SstPartitionerFixedPrefixFactory(0)); + return guard->get(); + }); + return 1; +} +} // namespace +#endif // ROCKSDB_LITE + +Status SstPartitionerFactory::CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result) { +#ifndef ROCKSDB_LITE + static std::once_flag once; + std::call_once(once, [&]() { + RegisterSstPartitionerFactories(*(ObjectLibrary::Default().get()), ""); + }); +#endif // ROCKSDB_LITE + return LoadSharedObject(options, value, nullptr, + result); +} } // namespace ROCKSDB_NAMESPACE diff --git a/db/db_table_properties_test.cc b/db/db_table_properties_test.cc index 4771f384d..c7618562d 100644 --- a/db/db_table_properties_test.cc +++ b/db/db_table_properties_test.cc @@ -94,6 +94,37 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) { VerifyTableProperties(db_, 10 + 11 + 12 + 13); } +TEST_F(DBTablePropertiesTest, CreateOnDeletionCollectorFactory) { + ConfigOptions options; + options.ignore_unsupported_options = false; + + std::shared_ptr factory; + std::string id = CompactOnDeletionCollectorFactory::kClassName(); + ASSERT_OK( + TablePropertiesCollectorFactory::CreateFromString(options, id, &factory)); + auto del_factory = factory->CheckedCast(); + ASSERT_NE(del_factory, nullptr); + ASSERT_EQ(0U, del_factory->GetWindowSize()); + ASSERT_EQ(0U, del_factory->GetDeletionTrigger()); + ASSERT_EQ(0.0, del_factory->GetDeletionRatio()); + ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString( + options, "window_size=100; deletion_trigger=90; id=" + id, &factory)); + del_factory = factory->CheckedCast(); + ASSERT_NE(del_factory, nullptr); + ASSERT_EQ(100U, del_factory->GetWindowSize()); + ASSERT_EQ(90U, del_factory->GetDeletionTrigger()); + ASSERT_EQ(0.0, del_factory->GetDeletionRatio()); + ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString( + options, + "window_size=100; deletion_trigger=90; deletion_ratio=0.5; id=" + id, + &factory)); + del_factory = factory->CheckedCast(); + ASSERT_NE(del_factory, nullptr); + ASSERT_EQ(100U, del_factory->GetWindowSize()); + ASSERT_EQ(90U, del_factory->GetDeletionTrigger()); + ASSERT_EQ(0.5, del_factory->GetDeletionRatio()); +} + TablePropertiesCollection DBTablePropertiesTest::TestGetPropertiesOfTablesInRange( std::vector ranges, std::size_t* num_properties, diff --git a/include/rocksdb/concurrent_task_limiter.h b/include/rocksdb/concurrent_task_limiter.h index f8e7ed8ab..760444e18 100644 --- a/include/rocksdb/concurrent_task_limiter.h +++ b/include/rocksdb/concurrent_task_limiter.h @@ -9,8 +9,11 @@ #pragma once -#include "rocksdb/env.h" -#include "rocksdb/statistics.h" +#include + +#include + +#include "rocksdb/rocksdb_namespace.h" namespace ROCKSDB_NAMESPACE { diff --git a/include/rocksdb/configurable.h b/include/rocksdb/configurable.h index de693eb9e..ee1afd87c 100644 --- a/include/rocksdb/configurable.h +++ b/include/rocksdb/configurable.h @@ -261,7 +261,6 @@ class Configurable { virtual Status ValidateOptions(const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const; - // Splits the input opt_value into the ID field and the remaining options. // The input opt_value can be in the form of "name" or "name=value // [;name=value]". The first form uses the "name" as an id with no options The diff --git a/include/rocksdb/file_checksum.h b/include/rocksdb/file_checksum.h index 00b2b9d57..50b6e8fdf 100644 --- a/include/rocksdb/file_checksum.h +++ b/include/rocksdb/file_checksum.h @@ -14,6 +14,7 @@ #include #include +#include "rocksdb/customizable.h" #include "rocksdb/status.h" namespace ROCKSDB_NAMESPACE { @@ -63,9 +64,13 @@ class FileChecksumGenerator { }; // Create the FileChecksumGenerator object for each SST file. -class FileChecksumGenFactory { +class FileChecksumGenFactory : public Customizable { public: virtual ~FileChecksumGenFactory() {} + static const char* Type() { return "FileChecksumGenFactory"; } + static Status CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result); // Create a new FileChecksumGenerator. virtual std::unique_ptr CreateFileChecksumGenerator( diff --git a/include/rocksdb/sst_partitioner.h b/include/rocksdb/sst_partitioner.h index 1ac16b49e..c8b8c39e5 100644 --- a/include/rocksdb/sst_partitioner.h +++ b/include/rocksdb/sst_partitioner.h @@ -9,6 +9,7 @@ #include #include +#include "rocksdb/customizable.h" #include "rocksdb/rocksdb_namespace.h" #include "rocksdb/slice.h" @@ -77,9 +78,13 @@ class SstPartitioner { }; }; -class SstPartitionerFactory { +class SstPartitionerFactory : public Customizable { public: virtual ~SstPartitionerFactory() {} + static const char* Type() { return "SstPartitionerFactory"; } + static Status CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result); virtual std::unique_ptr CreatePartitioner( const SstPartitioner::Context& context) const = 0; @@ -114,13 +119,12 @@ class SstPartitionerFixedPrefix : public SstPartitioner { */ class SstPartitionerFixedPrefixFactory : public SstPartitionerFactory { public: - explicit SstPartitionerFixedPrefixFactory(size_t len) : len_(len) {} + explicit SstPartitionerFixedPrefixFactory(size_t len); virtual ~SstPartitionerFixedPrefixFactory() {} - const char* Name() const override { - return "SstPartitionerFixedPrefixFactory"; - } + static const char* kClassName() { return "SstPartitionerFixedPrefixFactory"; } + const char* Name() const override { return kClassName(); } std::unique_ptr CreatePartitioner( const SstPartitioner::Context& /* context */) const override; diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index 1906e7acc..569f745f9 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -5,8 +5,12 @@ #pragma once #include + #include +#include #include + +#include "rocksdb/customizable.h" #include "rocksdb/status.h" #include "rocksdb/types.h" @@ -129,7 +133,7 @@ class TablePropertiesCollector { // Constructs TablePropertiesCollector. Internals create a new // TablePropertiesCollector for each new table -class TablePropertiesCollectorFactory { +class TablePropertiesCollectorFactory : public Customizable { public: struct Context { uint32_t column_family_id; @@ -137,6 +141,11 @@ class TablePropertiesCollectorFactory { }; virtual ~TablePropertiesCollectorFactory() {} + static const char* Type() { return "TablePropertiesCollectorFactory"; } + static Status CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result); + // has to be thread-safe virtual TablePropertiesCollector* CreateTablePropertiesCollector( TablePropertiesCollectorFactory::Context context) = 0; diff --git a/include/rocksdb/utilities/table_properties_collectors.h b/include/rocksdb/utilities/table_properties_collectors.h index 0f5612bce..f3a4ba005 100644 --- a/include/rocksdb/utilities/table_properties_collectors.h +++ b/include/rocksdb/utilities/table_properties_collectors.h @@ -19,6 +19,19 @@ namespace ROCKSDB_NAMESPACE { class CompactOnDeletionCollectorFactory : public TablePropertiesCollectorFactory { public: + // A factory of a table property collector that marks a SST + // file as need-compaction when it observe at least "D" deletion + // entries in any "N" consecutive entries, or the ratio of tombstone + // entries >= deletion_ratio. + // + // @param sliding_window_size "N" + // @param deletion_trigger "D" + // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction + // based on deletion ratio. + CompactOnDeletionCollectorFactory(size_t sliding_window_size, + size_t deletion_trigger, + double deletion_ratio); + ~CompactOnDeletionCollectorFactory() {} TablePropertiesCollector* CreateTablePropertiesCollector( @@ -29,12 +42,14 @@ class CompactOnDeletionCollectorFactory void SetWindowSize(size_t sliding_window_size) { sliding_window_size_.store(sliding_window_size); } + size_t GetWindowSize() const { return sliding_window_size_.load(); } // Change the value of deletion_trigger "D" void SetDeletionTrigger(size_t deletion_trigger) { deletion_trigger_.store(deletion_trigger); } + size_t GetDeletionTrigger() const { return deletion_trigger_.load(); } // Change deletion ratio. // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction // based on deletion ratio. @@ -42,33 +57,13 @@ class CompactOnDeletionCollectorFactory deletion_ratio_.store(deletion_ratio); } - const char* Name() const override { - return "CompactOnDeletionCollector"; - } + double GetDeletionRatio() const { return deletion_ratio_.load(); } + static const char* kClassName() { return "CompactOnDeletionCollector"; } + const char* Name() const override { return kClassName(); } std::string ToString() const override; private: - friend std::shared_ptr - NewCompactOnDeletionCollectorFactory(size_t sliding_window_size, - size_t deletion_trigger, - double deletion_ratio); - // A factory of a table property collector that marks a SST - // file as need-compaction when it observe at least "D" deletion - // entries in any "N" consecutive entries, or the ratio of tombstone - // entries >= deletion_ratio. - // - // @param sliding_window_size "N" - // @param deletion_trigger "D" - // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction - // based on deletion ratio. - CompactOnDeletionCollectorFactory(size_t sliding_window_size, - size_t deletion_trigger, - double deletion_ratio) - : sliding_window_size_(sliding_window_size), - deletion_trigger_(deletion_trigger), - deletion_ratio_(deletion_ratio) {} - std::atomic sliding_window_size_; std::atomic deletion_trigger_; std::atomic deletion_ratio_; diff --git a/include/rocksdb/wal_filter.h b/include/rocksdb/wal_filter.h index 98eddc2e2..109080cf0 100644 --- a/include/rocksdb/wal_filter.h +++ b/include/rocksdb/wal_filter.h @@ -8,17 +8,22 @@ #include #include +#include "rocksdb/customizable.h" #include "rocksdb/rocksdb_namespace.h" namespace ROCKSDB_NAMESPACE { class WriteBatch; +struct ConfigOptions; // WALFilter allows an application to inspect write-ahead-log (WAL) // records or modify their processing on recovery. // Please see the details below. -class WalFilter { +class WalFilter : public Customizable { public: + static const char* Type() { return "WalFilter"; } + static Status CreateFromString(const ConfigOptions& options, + const std::string& value, WalFilter** result); enum class WalProcessingOption { // Continue processing as usual kContinueProcessing = 0, diff --git a/options/cf_options.cc b/options/cf_options.cc index 3204f3b95..8da3930ce 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -673,6 +673,14 @@ static std::unordered_map return Status::NotFound("Mismatched table option: ", name); } }}}, + {"table_properties_collectors", + OptionTypeInfo::Vector< + std::shared_ptr>( + offset_of( + &ImmutableCFOptions::table_properties_collector_factories), + OptionVerificationType::kByName, OptionTypeFlags::kNone, + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kNone))}, {"compaction_filter", OptionTypeInfo::AsCustomRawPtr( offset_of(&ImmutableCFOptions::compaction_filter), @@ -694,6 +702,10 @@ static std::unordered_map {offset_of(&ImmutableCFOptions::compaction_pri), OptionType::kCompactionPri, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"sst_partitioner_factory", + OptionTypeInfo::AsCustomSharedPtr( + offset_of(&ImmutableCFOptions::sst_partitioner_factory), + OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)}, }; const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions"; diff --git a/options/customizable_test.cc b/options/customizable_test.cc index 470bf7dee..806a368a6 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -20,9 +20,11 @@ #include "port/stack_trace.h" #include "rocksdb/convenience.h" #include "rocksdb/env_encryption.h" +#include "rocksdb/file_checksum.h" #include "rocksdb/flush_block_policy.h" #include "rocksdb/secondary_cache.h" #include "rocksdb/slice_transform.h" +#include "rocksdb/sst_partitioner.h" #include "rocksdb/statistics.h" #include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/object_registry.h" @@ -32,6 +34,7 @@ #include "test_util/mock_time_env.h" #include "test_util/testharness.h" #include "test_util/testutil.h" +#include "util/file_checksum_helper.h" #include "util/string_util.h" #include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h" @@ -1297,6 +1300,41 @@ class MockCipher : public BlockCipher { Status Decrypt(char* data) override { return Encrypt(data); } }; +#endif // ROCKSDB_LITE + +class MockTablePropertiesCollectorFactory + : public TablePropertiesCollectorFactory { + private: + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return nullptr; + } + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } +}; + +class MockSstPartitionerFactory : public SstPartitionerFactory { + public: + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /* context */) const override { + return nullptr; + } +}; + +class MockFileChecksumGenFactory : public FileChecksumGenFactory { + public: + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } + std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& /*context*/) override { + return nullptr; + } +}; + +#ifndef ROCKSDB_LITE static int RegisterLocalObjects(ObjectLibrary& library, const std::string& /*arg*/) { size_t num_types; @@ -1367,6 +1405,33 @@ static int RegisterLocalObjects(ObjectLibrary& library, guard->reset(new TestSecondaryCache()); return guard->get(); }); + + library.Register( + MockSstPartitionerFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockSstPartitionerFactory()); + return guard->get(); + }); + + library.Register( + MockFileChecksumGenFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockFileChecksumGenFactory()); + return guard->get(); + }); + + library.Register( + MockTablePropertiesCollectorFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockTablePropertiesCollectorFactory()); + return guard->get(); + }); return static_cast(library.GetFactoryCount(&num_types)); } #endif // !ROCKSDB_LITE @@ -1443,6 +1508,58 @@ TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) { } } +#ifndef ROCKSDB_LITE +TEST_F(LoadCustomizableTest, LoadSstPartitionerFactoryTest) { + std::shared_ptr factory; + ASSERT_NOK(SstPartitionerFactory::CreateFromString(config_options_, "Mock", + &factory)); + ASSERT_OK(SstPartitionerFactory::CreateFromString( + config_options_, SstPartitionerFixedPrefixFactory::kClassName(), + &factory)); + ASSERT_NE(factory, nullptr); + ASSERT_STREQ(factory->Name(), SstPartitionerFixedPrefixFactory::kClassName()); + + if (RegisterTests("Test")) { + ASSERT_OK(SstPartitionerFactory::CreateFromString(config_options_, "Mock", + &factory)); + ASSERT_NE(factory, nullptr); + ASSERT_STREQ(factory->Name(), "Mock"); + } +} +#endif // ROCKSDB_LITE + +TEST_F(LoadCustomizableTest, LoadChecksumGenFactoryTest) { + std::shared_ptr factory; + ASSERT_NOK(FileChecksumGenFactory::CreateFromString(config_options_, "Mock", + &factory)); + ASSERT_OK(FileChecksumGenFactory::CreateFromString( + config_options_, FileChecksumGenCrc32cFactory::kClassName(), &factory)); + ASSERT_NE(factory, nullptr); + ASSERT_STREQ(factory->Name(), FileChecksumGenCrc32cFactory::kClassName()); + + if (RegisterTests("Test")) { + ASSERT_OK(FileChecksumGenFactory::CreateFromString(config_options_, "Mock", + &factory)); + ASSERT_NE(factory, nullptr); + ASSERT_STREQ(factory->Name(), "Mock"); + } +} + +TEST_F(LoadCustomizableTest, LoadTablePropertiesCollectorFactoryTest) { + std::shared_ptr factory; + ASSERT_NOK(TablePropertiesCollectorFactory::CreateFromString( + config_options_, MockTablePropertiesCollectorFactory::kClassName(), + &factory)); + if (RegisterTests("Test")) { + ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString( + config_options_, MockTablePropertiesCollectorFactory::kClassName(), + &factory)); + ASSERT_NE(factory, nullptr); + ASSERT_STREQ(factory->Name(), + MockTablePropertiesCollectorFactory::kClassName()); + } +} + TEST_F(LoadCustomizableTest, LoadComparatorTest) { const Comparator* bytewise = BytewiseComparator(); const Comparator* reverse = ReverseBytewiseComparator(); diff --git a/options/db_options.cc b/options/db_options.cc index e6e2c25ee..7a91c9e6d 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -171,6 +171,11 @@ static std::unordered_map {"allow_2pc", {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"wal_filter", + OptionTypeInfo::AsCustomRawPtr( + offsetof(struct ImmutableDBOptions, wal_filter), + OptionVerificationType::kByName, + (OptionTypeFlags::kAllowNull | OptionTypeFlags::kCompareNever))}, {"create_if_missing", {offsetof(struct ImmutableDBOptions, create_if_missing), OptionType::kBoolean, OptionVerificationType::kNormal, @@ -444,6 +449,10 @@ static std::unordered_map {offsetof(struct ImmutableDBOptions, allow_data_in_errors), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"file_checksum_gen_factory", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct ImmutableDBOptions, file_checksum_gen_factory), + OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)}, {"statistics", OptionTypeInfo::AsCustomSharedPtr( // Statistics should not be compared and can be null diff --git a/options/options_test.cc b/options/options_test.cc index 4de9c282d..2e3130284 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -19,6 +19,7 @@ #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/convenience.h" +#include "rocksdb/file_checksum.h" #include "rocksdb/memtablerep.h" #include "rocksdb/utilities/leveldb_options.h" #include "rocksdb/utilities/object_registry.h" @@ -1960,6 +1961,133 @@ TEST_F(OptionsTest, OnlyMutableCFOptions) { opt_str, &cf_opts)); delete cf_opts.compaction_filter; } + +TEST_F(OptionsTest, SstPartitionerTest) { + ConfigOptions cfg_opts; + ColumnFamilyOptions cf_opts, new_opt; + std::string opts_str, mismatch; + + ASSERT_OK(SstPartitionerFactory::CreateFromString( + cfg_opts, SstPartitionerFixedPrefixFactory::kClassName(), + &cf_opts.sst_partitioner_factory)); + ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr); + ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(), + SstPartitionerFixedPrefixFactory::kClassName()); + ASSERT_NOK(GetColumnFamilyOptionsFromString( + cfg_opts, ColumnFamilyOptions(), + std::string("sst_partitioner_factory={id=") + + SstPartitionerFixedPrefixFactory::kClassName() + "; unknown=10;}", + &cf_opts)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + cfg_opts, ColumnFamilyOptions(), + std::string("sst_partitioner_factory={id=") + + SstPartitionerFixedPrefixFactory::kClassName() + "; length=10;}", + &cf_opts)); + ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr); + ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(), + SstPartitionerFixedPrefixFactory::kClassName()); + ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, cf_opts, &opts_str)); + ASSERT_OK( + GetColumnFamilyOptionsFromString(cfg_opts, cf_opts, opts_str, &new_opt)); + ASSERT_NE(new_opt.sst_partitioner_factory, nullptr); + ASSERT_STREQ(new_opt.sst_partitioner_factory->Name(), + SstPartitionerFixedPrefixFactory::kClassName()); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, cf_opts, new_opt)); + ASSERT_TRUE(cf_opts.sst_partitioner_factory->AreEquivalent( + cfg_opts, new_opt.sst_partitioner_factory.get(), &mismatch)); +} + +TEST_F(OptionsTest, FileChecksumGenFactoryTest) { + ConfigOptions cfg_opts; + DBOptions db_opts, new_opt; + std::string opts_str, mismatch; + auto factory = GetFileChecksumGenCrc32cFactory(); + + cfg_opts.ignore_unsupported_options = false; + + ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str)); + ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt)); + + ASSERT_NE(factory, nullptr); + ASSERT_OK(FileChecksumGenFactory::CreateFromString( + cfg_opts, factory->Name(), &db_opts.file_checksum_gen_factory)); + ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr); + ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name()); + ASSERT_NOK(GetDBOptionsFromString( + cfg_opts, DBOptions(), "file_checksum_gen_factory=unknown", &db_opts)); + ASSERT_OK(GetDBOptionsFromString( + cfg_opts, DBOptions(), + std::string("file_checksum_gen_factory=") + factory->Name(), &db_opts)); + ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr); + ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name()); + + ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str)); + ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt)); + ASSERT_NE(new_opt.file_checksum_gen_factory, nullptr); + ASSERT_STREQ(new_opt.file_checksum_gen_factory->Name(), factory->Name()); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(cfg_opts, db_opts, new_opt)); + ASSERT_TRUE(factory->AreEquivalent( + cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch)); + ASSERT_TRUE(db_opts.file_checksum_gen_factory->AreEquivalent( + cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch)); +} + +class TestTablePropertiesCollectorFactory + : public TablePropertiesCollectorFactory { + private: + std::string id_; + + public: + explicit TestTablePropertiesCollectorFactory(const std::string& id) + : id_(id) {} + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return nullptr; + } + static const char* kClassName() { return "TestCollector"; } + const char* Name() const override { return kClassName(); } + std::string GetId() const override { + return std::string(kClassName()) + ":" + id_; + } +}; + +TEST_F(OptionsTest, OptionTablePropertiesTest) { + ConfigOptions cfg_opts; + ColumnFamilyOptions orig, copy; + orig.table_properties_collector_factories.push_back( + std::make_shared("1")); + orig.table_properties_collector_factories.push_back( + std::make_shared("2")); + + // Push two TablePropertiesCollectorFactories then create a new + // ColumnFamilyOptions based on those settings. The copy should + // have no properties but still match the original + std::string opts_str; + ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, orig, &opts_str)); + ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, ©)); + ASSERT_EQ(copy.table_properties_collector_factories.size(), 0); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy)); + + // Now register a TablePropertiesCollectorFactory + // Repeat the experiment. The copy should have the same + // properties as the original + cfg_opts.registry->AddLibrary("collector") + ->Register( + std::string(TestTablePropertiesCollectorFactory::kClassName()) + + ":.*", + [](const std::string& name, + std::unique_ptr* guard, + std::string* /* errmsg */) { + std::string id = name.substr( + strlen(TestTablePropertiesCollectorFactory::kClassName()) + 1); + guard->reset(new TestTablePropertiesCollectorFactory(id)); + return guard->get(); + }); + + ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, ©)); + ASSERT_EQ(copy.table_properties_collector_factories.size(), 2); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy)); +} #endif // !ROCKSDB_LITE TEST_F(OptionsTest, ConvertOptionsTest) { diff --git a/src.mk b/src.mk index 2dab0010b..571fc877f 100644 --- a/src.mk +++ b/src.mk @@ -285,6 +285,7 @@ LIB_SOURCES = \ utilities/transactions/write_unprepared_txn.cc \ utilities/transactions/write_unprepared_txn_db.cc \ utilities/ttl/db_ttl_impl.cc \ + utilities/wal_filter.cc \ utilities/write_batch_with_index/write_batch_with_index.cc \ utilities/write_batch_with_index/write_batch_with_index_internal.cc \ diff --git a/util/file_checksum_helper.cc b/util/file_checksum_helper.cc index 78ed524cb..2fcbf1e9e 100644 --- a/util/file_checksum_helper.cc +++ b/util/file_checksum_helper.cc @@ -15,6 +15,7 @@ #include "db/version_edit.h" #include "db/version_edit_handler.h" #include "file/sequence_file_reader.h" +#include "rocksdb/utilities/customizable_util.h" namespace ROCKSDB_NAMESPACE { @@ -133,4 +134,39 @@ Status GetFileChecksumsFromManifest(Env* src_env, const std::string& abs_path, return retriever.status(); } +#ifndef ROCKSDB_LITE +namespace { +static int RegisterFileChecksumGenFactories(ObjectLibrary& library, + const std::string& /*arg*/) { + library.Register( + FileChecksumGenCrc32cFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new FileChecksumGenCrc32cFactory()); + return guard->get(); + }); + return 1; +} +} // namespace +#endif // !ROCKSDB_LITE + +Status FileChecksumGenFactory::CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result) { +#ifndef ROCKSDB_LITE + static std::once_flag once; + std::call_once(once, [&]() { + RegisterFileChecksumGenFactories(*(ObjectLibrary::Default().get()), ""); + }); +#endif // ROCKSDB_LITE + if (value == FileChecksumGenCrc32cFactory::kClassName()) { + *result = GetFileChecksumGenCrc32cFactory(); + return Status::OK(); + } else { + Status s = LoadSharedObject(options, value, nullptr, + result); + return s; + } +} } // namespace ROCKSDB_NAMESPACE diff --git a/util/file_checksum_helper.h b/util/file_checksum_helper.h index 448927f55..42d32d4ad 100644 --- a/util/file_checksum_helper.h +++ b/util/file_checksum_helper.h @@ -58,7 +58,8 @@ class FileChecksumGenCrc32cFactory : public FileChecksumGenFactory { } } - const char* Name() const override { return "FileChecksumGenCrc32cFactory"; } + static const char* kClassName() { return "FileChecksumGenCrc32cFactory"; } + const char* Name() const override { return kClassName(); } }; // The default implementaion of FileChecksumList diff --git a/utilities/table_properties_collectors/compact_on_deletion_collector.cc b/utilities/table_properties_collectors/compact_on_deletion_collector.cc index 8e97118df..a05ef795d 100644 --- a/utilities/table_properties_collectors/compact_on_deletion_collector.cc +++ b/utilities/table_properties_collectors/compact_on_deletion_collector.cc @@ -3,14 +3,19 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -#ifndef ROCKSDB_LITE #include "utilities/table_properties_collectors/compact_on_deletion_collector.h" #include #include + +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" #include "rocksdb/utilities/table_properties_collectors.h" +#include "util/string_util.h" namespace ROCKSDB_NAMESPACE { +#ifndef ROCKSDB_LITE CompactOnDeletionCollector::CompactOnDeletionCollector( size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio) @@ -93,6 +98,74 @@ Status CompactOnDeletionCollector::Finish( finished_ = true; return Status::OK(); } +static std::unordered_map + on_deletion_collector_type_info = { +#ifndef ROCKSDB_LITE + {"window_size", + {0, OptionType::kUnknown, OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable, + [](const ConfigOptions&, const std::string&, const std::string& value, + void* addr) { + auto* factory = + static_cast(addr); + factory->SetWindowSize(ParseSizeT(value)); + return Status::OK(); + }, + [](const ConfigOptions&, const std::string&, const void* addr, + std::string* value) { + const auto* factory = + static_cast(addr); + *value = ToString(factory->GetWindowSize()); + return Status::OK(); + }, + nullptr}}, + {"deletion_trigger", + {0, OptionType::kUnknown, OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable, + [](const ConfigOptions&, const std::string&, const std::string& value, + void* addr) { + auto* factory = + static_cast(addr); + factory->SetDeletionTrigger(ParseSizeT(value)); + return Status::OK(); + }, + [](const ConfigOptions&, const std::string&, const void* addr, + std::string* value) { + const auto* factory = + static_cast(addr); + *value = ToString(factory->GetDeletionTrigger()); + return Status::OK(); + }, + nullptr}}, + {"deletion_ratio", + {0, OptionType::kUnknown, OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable, + [](const ConfigOptions&, const std::string&, const std::string& value, + void* addr) { + auto* factory = + static_cast(addr); + factory->SetDeletionRatio(ParseDouble(value)); + return Status::OK(); + }, + [](const ConfigOptions&, const std::string&, const void* addr, + std::string* value) { + const auto* factory = + static_cast(addr); + *value = ToString(factory->GetDeletionRatio()); + return Status::OK(); + }, + nullptr}}, + +#endif // ROCKSDB_LITE +}; + +CompactOnDeletionCollectorFactory::CompactOnDeletionCollectorFactory( + size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio) + : sliding_window_size_(sliding_window_size), + deletion_trigger_(deletion_trigger), + deletion_ratio_(deletion_ratio) { + RegisterOptions("", this, &on_deletion_collector_type_info); +} TablePropertiesCollector* CompactOnDeletionCollectorFactory::CreateTablePropertiesCollector( @@ -118,5 +191,37 @@ NewCompactOnDeletionCollectorFactory(size_t sliding_window_size, new CompactOnDeletionCollectorFactory(sliding_window_size, deletion_trigger, deletion_ratio)); } -} // namespace ROCKSDB_NAMESPACE +namespace { +static int RegisterTablePropertiesCollectorFactories( + ObjectLibrary& library, const std::string& /*arg*/) { + library.Register( + CompactOnDeletionCollectorFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + // By default, create a CompactionOnDeletionCollector that is disabled. + // Users will need to provide configuration parameters or call the + // corresponding Setter to enable the factory. + guard->reset(new CompactOnDeletionCollectorFactory(0, 0, 0)); + return guard->get(); + }); + return 1; +} +} // namespace #endif // !ROCKSDB_LITE + +Status TablePropertiesCollectorFactory::CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result) { +#ifndef ROCKSDB_LITE + static std::once_flag once; + std::call_once(once, [&]() { + RegisterTablePropertiesCollectorFactories(*(ObjectLibrary::Default().get()), + ""); + }); +#endif // ROCKSDB_LITE + return LoadSharedObject(options, value, + nullptr, result); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/utilities/wal_filter.cc b/utilities/wal_filter.cc new file mode 100644 index 000000000..98bba3610 --- /dev/null +++ b/utilities/wal_filter.cc @@ -0,0 +1,23 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/wal_filter.h" + +#include + +#include "rocksdb/convenience.h" +#include "rocksdb/options.h" +#include "rocksdb/utilities/customizable_util.h" + +namespace ROCKSDB_NAMESPACE { +Status WalFilter::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + WalFilter** filter) { + Status s = + LoadStaticObject(config_options, value, nullptr, filter); + return s; +} + +} // namespace ROCKSDB_NAMESPACE