Make SecondaryCache Customizable (#8480)

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8480

Reviewed By: zhichao-cao

Differential Revision: D29528740

Pulled By: mrambacher

fbshipit-source-id: fd0f70d15f66611c8498257a9973f7e98ca13839
main
mrambacher 3 years ago committed by Facebook GitHub Bot
parent 9dc887ece0
commit 570248aeff
  1. 1
      HISTORY.md
  2. 9
      cache/cache.cc
  3. 7
      cache/cache_bench_tool.cc
  4. 2
      cache/lru_cache_test.cc
  5. 5
      db_stress_tool/db_stress_test_base.cc
  6. 10
      include/rocksdb/secondary_cache.h
  7. 41
      options/customizable_test.cc
  8. 5
      tools/db_bench_tool.cc

@ -17,6 +17,7 @@
### Public API change ### Public API change
* Added APIs to the Customizable class to allow developers to create their own Customizable classes. Created the utilities/customizable_util.h file to contain helper methods for developing new Customizable classes. * Added APIs to the Customizable class to allow developers to create their own Customizable classes. Created the utilities/customizable_util.h file to contain helper methods for developing new Customizable classes.
* Change signature of SecondaryCache::Name(). Make SecondaryCache customizable and add SecondaryCache::CreateFromString method.
## 6.22.0 (2021-06-18) ## 6.22.0 (2021-06-18)
### Behavior Changes ### Behavior Changes
* Added two additional tickers, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH. These stats can be used to estimate the ratio of "garbage" (outdated) bytes in the memtable that are discarded at flush time. * Added two additional tickers, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH. These stats can be used to estimate the ratio of "garbage" (outdated) bytes in the memtable that are discarded at flush time.

9
cache/cache.cc vendored

@ -10,6 +10,8 @@
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "cache/lru_cache.h" #include "cache/lru_cache.h"
#include "rocksdb/secondary_cache.h"
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/options_type.h" #include "rocksdb/utilities/options_type.h"
#include "util/string_util.h" #include "util/string_util.h"
@ -34,6 +36,13 @@ static std::unordered_map<std::string, OptionTypeInfo>
}; };
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
Status SecondaryCache::CreateFromString(
const ConfigOptions& config_options, const std::string& value,
std::shared_ptr<SecondaryCache>* result) {
return LoadSharedObject<SecondaryCache>(config_options, value, nullptr,
result);
}
Status Cache::CreateFromString(const ConfigOptions& config_options, Status Cache::CreateFromString(const ConfigOptions& config_options,
const std::string& value, const std::string& value,
std::shared_ptr<Cache>* result) { std::shared_ptr<Cache>* result) {

@ -13,11 +13,11 @@
#include "monitoring/histogram.h" #include "monitoring/histogram.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/convenience.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/secondary_cache.h" #include "rocksdb/secondary_cache.h"
#include "rocksdb/system_clock.h" #include "rocksdb/system_clock.h"
#include "rocksdb/utilities/object_registry.h"
#include "table/block_based/cachable_entry.h" #include "table/block_based/cachable_entry.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/gflags_compat.h" #include "util/gflags_compat.h"
@ -233,9 +233,8 @@ class CacheBench {
LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5); LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5);
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
if (!FLAGS_secondary_cache_uri.empty()) { if (!FLAGS_secondary_cache_uri.empty()) {
Status s = Status s = SecondaryCache::CreateFromString(
ObjectRegistry::NewInstance()->NewSharedObject<SecondaryCache>( ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
FLAGS_secondary_cache_uri, &secondary_cache);
if (secondary_cache == nullptr) { if (secondary_cache == nullptr) {
fprintf( fprintf(
stderr, stderr,

@ -224,7 +224,7 @@ class TestSecondaryCache : public SecondaryCache {
} }
~TestSecondaryCache() override { cache_.reset(); } ~TestSecondaryCache() override { cache_.reset(); }
std::string Name() override { return "TestSecondaryCache"; } const char* Name() const override { return "TestSecondaryCache"; }
void InjectFailure() { inject_failure_ = true; } void InjectFailure() { inject_failure_ = true; }

@ -119,6 +119,7 @@ StressTest::~StressTest() {
std::shared_ptr<Cache> StressTest::NewCache(size_t capacity, std::shared_ptr<Cache> StressTest::NewCache(size_t capacity,
int32_t num_shard_bits) { int32_t num_shard_bits) {
ConfigOptions config_options;
if (capacity <= 0) { if (capacity <= 0) {
return nullptr; return nullptr;
} }
@ -136,8 +137,8 @@ std::shared_ptr<Cache> StressTest::NewCache(size_t capacity,
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
std::shared_ptr<SecondaryCache> secondary_cache; std::shared_ptr<SecondaryCache> secondary_cache;
if (!FLAGS_secondary_cache_uri.empty()) { if (!FLAGS_secondary_cache_uri.empty()) {
Status s = ObjectRegistry::NewInstance()->NewSharedObject<SecondaryCache>( Status s = SecondaryCache::CreateFromString(
FLAGS_secondary_cache_uri, &secondary_cache); config_options, FLAGS_secondary_cache_uri, &secondary_cache);
if (secondary_cache == nullptr) { if (secondary_cache == nullptr) {
fprintf(stderr, fprintf(stderr,
"No secondary cache registered matching string: %s status=%s\n", "No secondary cache registered matching string: %s status=%s\n",

@ -10,6 +10,7 @@
#include <string> #include <string>
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/customizable.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/statistics.h" #include "rocksdb/statistics.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
@ -42,13 +43,14 @@ class SecondaryCacheResultHandle {
// //
// Cache interface for caching blocks on a secondary tier (which can include // Cache interface for caching blocks on a secondary tier (which can include
// non-volatile media, or alternate forms of caching such as compressed data) // non-volatile media, or alternate forms of caching such as compressed data)
class SecondaryCache { class SecondaryCache : public Customizable {
public: public:
virtual ~SecondaryCache() {} virtual ~SecondaryCache() {}
virtual std::string Name() = 0; static const char* Type() { return "SecondaryCache"; }
static Status CreateFromString(const ConfigOptions& config_options,
static const std::string Type() { return "SecondaryCache"; } const std::string& id,
std::shared_ptr<SecondaryCache>* result);
// Insert the given value into this cache. The value is not written // Insert the given value into this cache. The value is not written
// directly. Rather, the SaveToCallback provided by helper_cb will be // directly. Rather, the SaveToCallback provided by helper_cb will be

@ -18,6 +18,7 @@
#include "options/options_helper.h" #include "options/options_helper.h"
#include "options/options_parser.h" #include "options/options_parser.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/secondary_cache.h"
#include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h" #include "rocksdb/utilities/options_type.h"
@ -870,6 +871,27 @@ TEST_F(CustomizableTest, MutableOptionsTest) {
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
class TestSecondaryCache : public SecondaryCache {
public:
const char* Name() const override { return kClassName(); }
static const char* kClassName() { return "Test"; }
Status Insert(const Slice& /*key*/, void* /*value*/,
const Cache::CacheItemHelper* /*helper*/) override {
return Status::NotSupported();
}
std::unique_ptr<SecondaryCacheResultHandle> Lookup(
const Slice& /*key*/, const Cache::CreateCallback& /*create_cb*/,
bool /*wait*/) override {
return nullptr;
}
void Erase(const Slice& /*key*/) override {}
// Wait for a collection of handles to become ready
void WaitAll(std::vector<SecondaryCacheResultHandle*> /*handles*/) override {}
std::string GetPrintableOptions() const override { return ""; }
};
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
// This method loads existing test classes into the ObjectRegistry // This method loads existing test classes into the ObjectRegistry
static int RegisterTestObjects(ObjectLibrary& library, static int RegisterTestObjects(ObjectLibrary& library,
@ -898,6 +920,13 @@ static int RegisterLocalObjects(ObjectLibrary& library,
const std::string& /*arg*/) { const std::string& /*arg*/) {
size_t num_types; size_t num_types;
// Load any locally defined objects here // Load any locally defined objects here
library.Register<SecondaryCache>(
TestSecondaryCache::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<SecondaryCache>* guard,
std::string* /* errmsg */) {
guard->reset(new TestSecondaryCache());
return guard->get();
});
return static_cast<int>(library.GetFactoryCount(&num_types)); return static_cast<int>(library.GetFactoryCount(&num_types));
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
@ -941,6 +970,18 @@ TEST_F(LoadCustomizableTest, LoadTableFactoryTest) {
} }
} }
TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) {
std::shared_ptr<SecondaryCache> result;
ASSERT_NOK(SecondaryCache::CreateFromString(
config_options_, TestSecondaryCache::kClassName(), &result));
if (RegisterTests("Test")) {
ASSERT_OK(SecondaryCache::CreateFromString(
config_options_, TestSecondaryCache::kClassName(), &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), TestSecondaryCache::kClassName());
}
}
TEST_F(LoadCustomizableTest, LoadComparatorTest) { TEST_F(LoadCustomizableTest, LoadComparatorTest) {
const Comparator* bytewise = BytewiseComparator(); const Comparator* bytewise = BytewiseComparator();
const Comparator* reverse = ReverseBytewiseComparator(); const Comparator* reverse = ReverseBytewiseComparator();

@ -2869,9 +2869,8 @@ class Benchmark {
} }
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
if (!FLAGS_secondary_cache_uri.empty()) { if (!FLAGS_secondary_cache_uri.empty()) {
Status s = Status s = SecondaryCache::CreateFromString(
ObjectRegistry::NewInstance()->NewSharedObject<SecondaryCache>( ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
FLAGS_secondary_cache_uri, &secondary_cache);
if (secondary_cache == nullptr) { if (secondary_cache == nullptr) {
fprintf( fprintf(
stderr, stderr,

Loading…
Cancel
Save