From 570248aeff8873acdc8ebbad57962d02a1078aeb Mon Sep 17 00:00:00 2001 From: mrambacher Date: Tue, 6 Jul 2021 09:17:13 -0700 Subject: [PATCH] Make SecondaryCache Customizable (#8480) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8480 Reviewed By: zhichao-cao Differential Revision: D29528740 Pulled By: mrambacher fbshipit-source-id: fd0f70d15f66611c8498257a9973f7e98ca13839 --- HISTORY.md | 1 + cache/cache.cc | 9 ++++++ cache/cache_bench_tool.cc | 7 ++--- cache/lru_cache_test.cc | 2 +- db_stress_tool/db_stress_test_base.cc | 5 ++-- include/rocksdb/secondary_cache.h | 10 ++++--- options/customizable_test.cc | 41 +++++++++++++++++++++++++++ tools/db_bench_tool.cc | 5 ++-- 8 files changed, 66 insertions(+), 14 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index d335be8cb..e6304896a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -17,6 +17,7 @@ ### Public API change * Added APIs to the Customizable class to allow developers to create their own Customizable classes. Created the utilities/customizable_util.h file to contain helper methods for developing new Customizable classes. +* Change signature of SecondaryCache::Name(). Make SecondaryCache customizable and add SecondaryCache::CreateFromString method. ## 6.22.0 (2021-06-18) ### Behavior Changes * Added two additional tickers, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH. These stats can be used to estimate the ratio of "garbage" (outdated) bytes in the memtable that are discarded at flush time. diff --git a/cache/cache.cc b/cache/cache.cc index 4eef1c2d6..a7d9f86bf 100644 --- a/cache/cache.cc +++ b/cache/cache.cc @@ -10,6 +10,8 @@ #include "rocksdb/cache.h" #include "cache/lru_cache.h" +#include "rocksdb/secondary_cache.h" +#include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/options_type.h" #include "util/string_util.h" @@ -34,6 +36,13 @@ static std::unordered_map }; #endif // ROCKSDB_LITE +Status SecondaryCache::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + return LoadSharedObject(config_options, value, nullptr, + result); +} + Status Cache::CreateFromString(const ConfigOptions& config_options, const std::string& value, std::shared_ptr* result) { diff --git a/cache/cache_bench_tool.cc b/cache/cache_bench_tool.cc index 9fadf85a9..4d8939041 100644 --- a/cache/cache_bench_tool.cc +++ b/cache/cache_bench_tool.cc @@ -13,11 +13,11 @@ #include "monitoring/histogram.h" #include "port/port.h" #include "rocksdb/cache.h" +#include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/secondary_cache.h" #include "rocksdb/system_clock.h" -#include "rocksdb/utilities/object_registry.h" #include "table/block_based/cachable_entry.h" #include "util/coding.h" #include "util/gflags_compat.h" @@ -233,9 +233,8 @@ class CacheBench { LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5); #ifndef ROCKSDB_LITE if (!FLAGS_secondary_cache_uri.empty()) { - Status s = - ObjectRegistry::NewInstance()->NewSharedObject( - FLAGS_secondary_cache_uri, &secondary_cache); + Status s = SecondaryCache::CreateFromString( + ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache); if (secondary_cache == nullptr) { fprintf( stderr, diff --git a/cache/lru_cache_test.cc b/cache/lru_cache_test.cc index d20fd2463..a3688b8a2 100644 --- a/cache/lru_cache_test.cc +++ b/cache/lru_cache_test.cc @@ -224,7 +224,7 @@ class TestSecondaryCache : public SecondaryCache { } ~TestSecondaryCache() override { cache_.reset(); } - std::string Name() override { return "TestSecondaryCache"; } + const char* Name() const override { return "TestSecondaryCache"; } void InjectFailure() { inject_failure_ = true; } diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index af34f579f..f96f31fbc 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -119,6 +119,7 @@ StressTest::~StressTest() { std::shared_ptr StressTest::NewCache(size_t capacity, int32_t num_shard_bits) { + ConfigOptions config_options; if (capacity <= 0) { return nullptr; } @@ -136,8 +137,8 @@ std::shared_ptr StressTest::NewCache(size_t capacity, #ifndef ROCKSDB_LITE std::shared_ptr secondary_cache; if (!FLAGS_secondary_cache_uri.empty()) { - Status s = ObjectRegistry::NewInstance()->NewSharedObject( - FLAGS_secondary_cache_uri, &secondary_cache); + Status s = SecondaryCache::CreateFromString( + config_options, FLAGS_secondary_cache_uri, &secondary_cache); if (secondary_cache == nullptr) { fprintf(stderr, "No secondary cache registered matching string: %s status=%s\n", diff --git a/include/rocksdb/secondary_cache.h b/include/rocksdb/secondary_cache.h index 221b3e5f2..20632d2ff 100644 --- a/include/rocksdb/secondary_cache.h +++ b/include/rocksdb/secondary_cache.h @@ -10,6 +10,7 @@ #include #include "rocksdb/cache.h" +#include "rocksdb/customizable.h" #include "rocksdb/slice.h" #include "rocksdb/statistics.h" #include "rocksdb/status.h" @@ -42,13 +43,14 @@ class SecondaryCacheResultHandle { // // Cache interface for caching blocks on a secondary tier (which can include // non-volatile media, or alternate forms of caching such as compressed data) -class SecondaryCache { +class SecondaryCache : public Customizable { public: virtual ~SecondaryCache() {} - virtual std::string Name() = 0; - - static const std::string Type() { return "SecondaryCache"; } + static const char* Type() { return "SecondaryCache"; } + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* result); // Insert the given value into this cache. The value is not written // directly. Rather, the SaveToCallback provided by helper_cb will be diff --git a/options/customizable_test.cc b/options/customizable_test.cc index 04f5761fa..2098f675e 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -18,6 +18,7 @@ #include "options/options_helper.h" #include "options/options_parser.h" #include "rocksdb/convenience.h" +#include "rocksdb/secondary_cache.h" #include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/options_type.h" @@ -870,6 +871,27 @@ TEST_F(CustomizableTest, MutableOptionsTest) { } #endif // !ROCKSDB_LITE +class TestSecondaryCache : public SecondaryCache { + public: + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "Test"; } + Status Insert(const Slice& /*key*/, void* /*value*/, + const Cache::CacheItemHelper* /*helper*/) override { + return Status::NotSupported(); + } + std::unique_ptr Lookup( + const Slice& /*key*/, const Cache::CreateCallback& /*create_cb*/, + bool /*wait*/) override { + return nullptr; + } + void Erase(const Slice& /*key*/) override {} + + // Wait for a collection of handles to become ready + void WaitAll(std::vector /*handles*/) override {} + + std::string GetPrintableOptions() const override { return ""; } +}; + #ifndef ROCKSDB_LITE // This method loads existing test classes into the ObjectRegistry static int RegisterTestObjects(ObjectLibrary& library, @@ -898,6 +920,13 @@ static int RegisterLocalObjects(ObjectLibrary& library, const std::string& /*arg*/) { size_t num_types; // Load any locally defined objects here + library.Register( + TestSecondaryCache::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TestSecondaryCache()); + return guard->get(); + }); return static_cast(library.GetFactoryCount(&num_types)); } #endif // !ROCKSDB_LITE @@ -941,6 +970,18 @@ TEST_F(LoadCustomizableTest, LoadTableFactoryTest) { } } +TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) { + std::shared_ptr result; + ASSERT_NOK(SecondaryCache::CreateFromString( + config_options_, TestSecondaryCache::kClassName(), &result)); + if (RegisterTests("Test")) { + ASSERT_OK(SecondaryCache::CreateFromString( + config_options_, TestSecondaryCache::kClassName(), &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), TestSecondaryCache::kClassName()); + } +} + TEST_F(LoadCustomizableTest, LoadComparatorTest) { const Comparator* bytewise = BytewiseComparator(); const Comparator* reverse = ReverseBytewiseComparator(); diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index e4f07f290..b4d5761be 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -2869,9 +2869,8 @@ class Benchmark { } #ifndef ROCKSDB_LITE if (!FLAGS_secondary_cache_uri.empty()) { - Status s = - ObjectRegistry::NewInstance()->NewSharedObject( - FLAGS_secondary_cache_uri, &secondary_cache); + Status s = SecondaryCache::CreateFromString( + ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache); if (secondary_cache == nullptr) { fprintf( stderr,