fork of https://github.com/oxigraph/rocksdb and https://github.com/facebook/rocksdb for nextgraph and oxigraph
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
598 lines
22 KiB
598 lines
22 KiB
3 years ago
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||
|
|
// This source code is licensed under both the GPLv2 (found in the
|
||
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
||
|
|
// (found in the LICENSE.Apache file in the root directory).
|
||
|
|
|
||
|
|
#include "cache/lru_secondary_cache.h"
|
||
|
|
|
||
|
|
#include <algorithm>
|
||
|
|
#include <cstdint>
|
||
|
|
|
||
|
|
#include "memory/jemalloc_nodump_allocator.h"
|
||
|
|
#include "memory/memory_allocator.h"
|
||
|
|
#include "test_util/testharness.h"
|
||
|
|
#include "test_util/testutil.h"
|
||
|
|
#include "util/compression.h"
|
||
|
|
#include "util/random.h"
|
||
|
|
|
||
|
|
namespace ROCKSDB_NAMESPACE {
|
||
|
|
|
||
|
|
class LRUSecondaryCacheTest : public testing::Test {
|
||
|
|
public:
|
||
|
|
LRUSecondaryCacheTest() : fail_create_(false) {}
|
||
|
|
~LRUSecondaryCacheTest() {}
|
||
|
|
|
||
|
|
protected:
|
||
|
|
class TestItem {
|
||
|
|
public:
|
||
|
|
TestItem(const char* buf, size_t size) : buf_(new char[size]), size_(size) {
|
||
|
|
memcpy(buf_.get(), buf, size);
|
||
|
|
}
|
||
|
|
~TestItem() {}
|
||
|
|
|
||
|
|
char* Buf() { return buf_.get(); }
|
||
|
|
size_t Size() { return size_; }
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::unique_ptr<char[]> buf_;
|
||
|
|
size_t size_;
|
||
|
|
};
|
||
|
|
|
||
|
|
static size_t SizeCallback(void* obj) {
|
||
|
|
return reinterpret_cast<TestItem*>(obj)->Size();
|
||
|
|
}
|
||
|
|
|
||
|
|
static Status SaveToCallback(void* from_obj, size_t from_offset,
|
||
|
|
size_t length, void* out) {
|
||
|
|
TestItem* item = reinterpret_cast<TestItem*>(from_obj);
|
||
|
|
const char* buf = item->Buf();
|
||
|
|
EXPECT_EQ(length, item->Size());
|
||
|
|
EXPECT_EQ(from_offset, 0);
|
||
|
|
memcpy(out, buf, length);
|
||
|
|
return Status::OK();
|
||
|
|
}
|
||
|
|
|
||
|
|
static void DeletionCallback(const Slice& /*key*/, void* obj) {
|
||
|
|
delete reinterpret_cast<TestItem*>(obj);
|
||
|
|
obj = nullptr;
|
||
|
|
}
|
||
|
|
|
||
|
|
static Cache::CacheItemHelper helper_;
|
||
|
|
|
||
|
|
static Status SaveToCallbackFail(void* /*obj*/, size_t /*offset*/,
|
||
|
|
size_t /*size*/, void* /*out*/) {
|
||
|
|
return Status::NotSupported();
|
||
|
|
}
|
||
|
|
|
||
|
|
static Cache::CacheItemHelper helper_fail_;
|
||
|
|
|
||
|
|
Cache::CreateCallback test_item_creator = [&](const void* buf, size_t size,
|
||
|
|
void** out_obj,
|
||
|
|
size_t* charge) -> Status {
|
||
|
|
if (fail_create_) {
|
||
|
|
return Status::NotSupported();
|
||
|
|
}
|
||
|
|
*out_obj = reinterpret_cast<void*>(new TestItem((char*)buf, size));
|
||
|
|
*charge = size;
|
||
|
|
return Status::OK();
|
||
|
|
};
|
||
|
|
|
||
|
|
void SetFailCreate(bool fail) { fail_create_ = fail; }
|
||
|
|
|
||
|
|
void BasicTest(bool sec_cache_is_compressed, bool use_jemalloc) {
|
||
|
|
LRUSecondaryCacheOptions opts;
|
||
|
|
opts.capacity = 2048;
|
||
|
|
opts.num_shard_bits = 0;
|
||
|
|
opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
||
|
|
|
||
|
|
if (sec_cache_is_compressed) {
|
||
|
|
if (!LZ4_Supported()) {
|
||
|
|
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
|
||
|
|
opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (use_jemalloc) {
|
||
|
|
JemallocAllocatorOptions jopts;
|
||
|
|
std::shared_ptr<MemoryAllocator> allocator;
|
||
|
|
std::string msg;
|
||
|
|
if (JemallocNodumpAllocator::IsSupported(&msg)) {
|
||
|
|
Status s = NewJemallocNodumpAllocator(jopts, &allocator);
|
||
|
|
if (s.ok()) {
|
||
|
|
opts.memory_allocator = allocator;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
std::shared_ptr<SecondaryCache> cache = NewLRUSecondaryCache(opts);
|
||
|
|
|
||
|
|
// Lookup an non-existent key.
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle0 =
|
||
|
|
cache->Lookup("k0", test_item_creator, true);
|
||
|
|
ASSERT_EQ(handle0, nullptr);
|
||
|
|
|
||
|
|
Random rnd(301);
|
||
|
|
// Insert and Lookup the first item.
|
||
|
|
std::string str1;
|
||
|
|
test::CompressibleString(&rnd, 0.25, 1000, &str1);
|
||
|
|
TestItem item1(str1.data(), str1.length());
|
||
|
|
ASSERT_OK(cache->Insert("k1", &item1, &LRUSecondaryCacheTest::helper_));
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle1 =
|
||
|
|
cache->Lookup("k1", test_item_creator, true);
|
||
|
|
ASSERT_NE(handle1, nullptr);
|
||
|
|
// delete reinterpret_cast<TestItem*>(handle1->Value());
|
||
|
|
std::unique_ptr<TestItem> val1 =
|
||
|
|
std::unique_ptr<TestItem>(static_cast<TestItem*>(handle1->Value()));
|
||
|
|
ASSERT_NE(val1, nullptr);
|
||
|
|
ASSERT_EQ(memcmp(val1->Buf(), item1.Buf(), item1.Size()), 0);
|
||
|
|
|
||
|
|
// Insert and Lookup the second item.
|
||
|
|
std::string str2;
|
||
|
|
test::CompressibleString(&rnd, 0.5, 1000, &str2);
|
||
|
|
TestItem item2(str2.data(), str2.length());
|
||
|
|
ASSERT_OK(cache->Insert("k2", &item2, &LRUSecondaryCacheTest::helper_));
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle2 =
|
||
|
|
cache->Lookup("k2", test_item_creator, true);
|
||
|
|
ASSERT_NE(handle2, nullptr);
|
||
|
|
std::unique_ptr<TestItem> val2 =
|
||
|
|
std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2->Value()));
|
||
|
|
ASSERT_NE(val2, nullptr);
|
||
|
|
ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);
|
||
|
|
|
||
|
|
// Lookup the first item again to make sure it is still in the cache.
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle1_1 =
|
||
|
|
cache->Lookup("k1", test_item_creator, true);
|
||
|
|
ASSERT_NE(handle1_1, nullptr);
|
||
|
|
std::unique_ptr<TestItem> val1_1 =
|
||
|
|
std::unique_ptr<TestItem>(static_cast<TestItem*>(handle1_1->Value()));
|
||
|
|
ASSERT_NE(val1_1, nullptr);
|
||
|
|
ASSERT_EQ(memcmp(val1_1->Buf(), item1.Buf(), item1.Size()), 0);
|
||
|
|
|
||
|
|
std::vector<SecondaryCacheResultHandle*> handles = {handle1.get(),
|
||
|
|
handle2.get()};
|
||
|
|
cache->WaitAll(handles);
|
||
|
|
|
||
|
|
cache->Erase("k1");
|
||
|
|
handle1 = cache->Lookup("k1", test_item_creator, true);
|
||
|
|
ASSERT_EQ(handle1, nullptr);
|
||
|
|
|
||
|
|
cache.reset();
|
||
|
|
}
|
||
|
|
|
||
|
|
void FailsTest(bool sec_cache_is_compressed) {
|
||
|
|
LRUSecondaryCacheOptions secondary_cache_opts;
|
||
|
|
if (sec_cache_is_compressed) {
|
||
|
|
if (!LZ4_Supported()) {
|
||
|
|
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
|
||
|
|
secondary_cache_opts.capacity = 1100;
|
||
|
|
secondary_cache_opts.num_shard_bits = 0;
|
||
|
|
secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
||
|
|
std::shared_ptr<SecondaryCache> cache =
|
||
|
|
NewLRUSecondaryCache(secondary_cache_opts);
|
||
|
|
|
||
|
|
// Insert and Lookup the first item.
|
||
|
|
Random rnd(301);
|
||
|
|
std::string str1(rnd.RandomString(1000));
|
||
|
|
TestItem item1(str1.data(), str1.length());
|
||
|
|
ASSERT_OK(cache->Insert("k1", &item1, &LRUSecondaryCacheTest::helper_));
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle1 =
|
||
|
|
cache->Lookup("k1", test_item_creator, true);
|
||
|
|
ASSERT_NE(handle1, nullptr);
|
||
|
|
std::unique_ptr<TestItem> val1 =
|
||
|
|
std::unique_ptr<TestItem>(static_cast<TestItem*>(handle1->Value()));
|
||
|
|
ASSERT_NE(val1, nullptr);
|
||
|
|
ASSERT_EQ(memcmp(val1->Buf(), item1.Buf(), item1.Size()), 0);
|
||
|
|
|
||
|
|
// Insert and Lookup the second item.
|
||
|
|
std::string str2(rnd.RandomString(200));
|
||
|
|
TestItem item2(str2.data(), str2.length());
|
||
|
|
// k1 is evicted.
|
||
|
|
ASSERT_OK(cache->Insert("k2", &item2, &LRUSecondaryCacheTest::helper_));
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle1_1 =
|
||
|
|
cache->Lookup("k1", test_item_creator, true);
|
||
|
|
ASSERT_EQ(handle1_1, nullptr);
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle2 =
|
||
|
|
cache->Lookup("k2", test_item_creator, true);
|
||
|
|
ASSERT_NE(handle2, nullptr);
|
||
|
|
std::unique_ptr<TestItem> val2 =
|
||
|
|
std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2->Value()));
|
||
|
|
ASSERT_NE(val2, nullptr);
|
||
|
|
ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);
|
||
|
|
|
||
|
|
// Create Fails.
|
||
|
|
SetFailCreate(true);
|
||
|
|
std::unique_ptr<SecondaryCacheResultHandle> handle2_1 =
|
||
|
|
cache->Lookup("k2", test_item_creator, true);
|
||
|
|
ASSERT_EQ(handle2_1, nullptr);
|
||
|
|
|
||
|
|
// Save Fails.
|
||
|
|
std::string str3 = rnd.RandomString(10);
|
||
|
|
TestItem item3(str3.data(), str3.length());
|
||
|
|
ASSERT_NOK(
|
||
|
|
cache->Insert("k3", &item3, &LRUSecondaryCacheTest::helper_fail_));
|
||
|
|
|
||
|
|
cache.reset();
|
||
|
|
}
|
||
|
|
|
||
|
|
void BasicIntegrationTest(bool sec_cache_is_compressed) {
|
||
|
|
LRUSecondaryCacheOptions secondary_cache_opts;
|
||
|
|
|
||
|
|
if (sec_cache_is_compressed) {
|
||
|
|
if (!LZ4_Supported()) {
|
||
|
|
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
|
||
|
|
secondary_cache_opts.capacity = 2300;
|
||
|
|
secondary_cache_opts.num_shard_bits = 0;
|
||
|
|
secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
||
|
|
std::shared_ptr<SecondaryCache> secondary_cache =
|
||
|
|
NewLRUSecondaryCache(secondary_cache_opts);
|
||
|
|
LRUCacheOptions lru_cache_opts(1024, 0, false, 0.5, nullptr,
|
||
|
|
kDefaultToAdaptiveMutex,
|
||
|
|
kDontChargeCacheMetadata);
|
||
|
|
lru_cache_opts.secondary_cache = secondary_cache;
|
||
|
|
std::shared_ptr<Cache> cache = NewLRUCache(lru_cache_opts);
|
||
|
|
std::shared_ptr<Statistics> stats = CreateDBStatistics();
|
||
|
|
|
||
|
|
Random rnd(301);
|
||
|
|
|
||
|
|
std::string str1 = rnd.RandomString(1010);
|
||
|
|
std::string str1_clone{str1};
|
||
|
|
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
||
|
|
ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_,
|
||
|
|
str1.length()));
|
||
|
|
|
||
|
|
std::string str2 = rnd.RandomString(1020);
|
||
|
|
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
||
|
|
// After Insert, lru cache contains k2 and secondary cache contains k1.
|
||
|
|
ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_,
|
||
|
|
str2.length()));
|
||
|
|
|
||
|
|
std::string str3 = rnd.RandomString(1020);
|
||
|
|
TestItem* item3 = new TestItem(str3.data(), str3.length());
|
||
|
|
// After Insert, lru cache contains k3 and secondary cache contains k1 and
|
||
|
|
// k2
|
||
|
|
ASSERT_OK(cache->Insert("k3", item3, &LRUSecondaryCacheTest::helper_,
|
||
|
|
str3.length()));
|
||
|
|
|
||
|
|
Cache::Handle* handle;
|
||
|
|
handle =
|
||
|
|
cache->Lookup("k3", &LRUSecondaryCacheTest::helper_, test_item_creator,
|
||
|
|
Cache::Priority::LOW, true, stats.get());
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
TestItem* val3 = static_cast<TestItem*>(cache->Value(handle));
|
||
|
|
ASSERT_NE(val3, nullptr);
|
||
|
|
ASSERT_EQ(memcmp(val3->Buf(), item3->Buf(), item3->Size()), 0);
|
||
|
|
cache->Release(handle);
|
||
|
|
|
||
|
|
// Lookup an non-existent key.
|
||
|
|
handle =
|
||
|
|
cache->Lookup("k0", &LRUSecondaryCacheTest::helper_, test_item_creator,
|
||
|
|
Cache::Priority::LOW, true, stats.get());
|
||
|
|
ASSERT_EQ(handle, nullptr);
|
||
|
|
|
||
|
|
// This Lookup should promote k1 and demote k3, so k2 is evicted from the
|
||
|
|
// secondary cache. The lru cache contains k1 and secondary cache contains
|
||
|
|
// k3. item1 was Free(), so it cannot be compared against the item1.
|
||
|
|
handle =
|
||
|
|
cache->Lookup("k1", &LRUSecondaryCacheTest::helper_, test_item_creator,
|
||
|
|
Cache::Priority::LOW, true, stats.get());
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
TestItem* val1_1 = static_cast<TestItem*>(cache->Value(handle));
|
||
|
|
ASSERT_NE(val1_1, nullptr);
|
||
|
|
ASSERT_EQ(memcmp(val1_1->Buf(), str1_clone.data(), str1_clone.size()), 0);
|
||
|
|
cache->Release(handle);
|
||
|
|
|
||
|
|
handle =
|
||
|
|
cache->Lookup("k2", &LRUSecondaryCacheTest::helper_, test_item_creator,
|
||
|
|
Cache::Priority::LOW, true, stats.get());
|
||
|
|
ASSERT_EQ(handle, nullptr);
|
||
|
|
|
||
|
|
cache.reset();
|
||
|
|
secondary_cache.reset();
|
||
|
|
}
|
||
|
|
|
||
|
|
void BasicIntegrationFailTest(bool sec_cache_is_compressed) {
|
||
|
|
LRUSecondaryCacheOptions secondary_cache_opts;
|
||
|
|
|
||
|
|
if (sec_cache_is_compressed) {
|
||
|
|
if (!LZ4_Supported()) {
|
||
|
|
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
|
||
|
|
secondary_cache_opts.capacity = 2048;
|
||
|
|
secondary_cache_opts.num_shard_bits = 0;
|
||
|
|
secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
||
|
|
std::shared_ptr<SecondaryCache> secondary_cache =
|
||
|
|
NewLRUSecondaryCache(secondary_cache_opts);
|
||
|
|
|
||
|
|
LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
|
||
|
|
kDontChargeCacheMetadata);
|
||
|
|
opts.secondary_cache = secondary_cache;
|
||
|
|
std::shared_ptr<Cache> cache = NewLRUCache(opts);
|
||
|
|
|
||
|
|
Random rnd(301);
|
||
|
|
std::string str1 = rnd.RandomString(1020);
|
||
|
|
auto item1 =
|
||
|
|
std::unique_ptr<TestItem>(new TestItem(str1.data(), str1.length()));
|
||
|
|
ASSERT_NOK(cache->Insert("k1", item1.get(), nullptr, str1.length()));
|
||
|
|
ASSERT_OK(cache->Insert("k1", item1.get(), &LRUSecondaryCacheTest::helper_,
|
||
|
|
str1.length()));
|
||
|
|
item1.release(); // Appease clang-analyze "potential memory leak"
|
||
|
|
|
||
|
|
Cache::Handle* handle;
|
||
|
|
handle = cache->Lookup("k2", nullptr, test_item_creator,
|
||
|
|
Cache::Priority::LOW, true);
|
||
|
|
ASSERT_EQ(handle, nullptr);
|
||
|
|
handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, false);
|
||
|
|
ASSERT_EQ(handle, nullptr);
|
||
|
|
|
||
|
|
cache.reset();
|
||
|
|
secondary_cache.reset();
|
||
|
|
}
|
||
|
|
|
||
|
|
void IntegrationSaveFailTest(bool sec_cache_is_compressed) {
|
||
|
|
LRUSecondaryCacheOptions secondary_cache_opts;
|
||
|
|
|
||
|
|
if (sec_cache_is_compressed) {
|
||
|
|
if (!LZ4_Supported()) {
|
||
|
|
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
|
||
|
|
secondary_cache_opts.capacity = 2048;
|
||
|
|
secondary_cache_opts.num_shard_bits = 0;
|
||
|
|
secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
||
|
|
|
||
|
|
std::shared_ptr<SecondaryCache> secondary_cache =
|
||
|
|
NewLRUSecondaryCache(secondary_cache_opts);
|
||
|
|
|
||
|
|
LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
|
||
|
|
kDontChargeCacheMetadata);
|
||
|
|
opts.secondary_cache = secondary_cache;
|
||
|
|
std::shared_ptr<Cache> cache = NewLRUCache(opts);
|
||
|
|
|
||
|
|
Random rnd(301);
|
||
|
|
std::string str1 = rnd.RandomString(1020);
|
||
|
|
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
||
|
|
ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_fail_,
|
||
|
|
str1.length()));
|
||
|
|
std::string str2 = rnd.RandomString(1020);
|
||
|
|
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
||
|
|
// k1 should be demoted to the secondary cache.
|
||
|
|
ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_fail_,
|
||
|
|
str2.length()));
|
||
|
|
|
||
|
|
Cache::Handle* handle;
|
||
|
|
handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_fail_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
cache->Release(handle);
|
||
|
|
// This lookup should fail, since k1 demotion would have failed
|
||
|
|
handle = cache->Lookup("k1", &LRUSecondaryCacheTest::helper_fail_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_EQ(handle, nullptr);
|
||
|
|
// Since k1 didn't get promoted, k2 should still be in cache
|
||
|
|
handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_fail_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
cache->Release(handle);
|
||
|
|
|
||
|
|
cache.reset();
|
||
|
|
secondary_cache.reset();
|
||
|
|
}
|
||
|
|
|
||
|
|
void IntegrationCreateFailTest(bool sec_cache_is_compressed) {
|
||
|
|
LRUSecondaryCacheOptions secondary_cache_opts;
|
||
|
|
|
||
|
|
if (sec_cache_is_compressed) {
|
||
|
|
if (!LZ4_Supported()) {
|
||
|
|
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
|
||
|
|
secondary_cache_opts.capacity = 2048;
|
||
|
|
secondary_cache_opts.num_shard_bits = 0;
|
||
|
|
secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
||
|
|
|
||
|
|
std::shared_ptr<SecondaryCache> secondary_cache =
|
||
|
|
NewLRUSecondaryCache(secondary_cache_opts);
|
||
|
|
|
||
|
|
LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
|
||
|
|
kDontChargeCacheMetadata);
|
||
|
|
opts.secondary_cache = secondary_cache;
|
||
|
|
std::shared_ptr<Cache> cache = NewLRUCache(opts);
|
||
|
|
|
||
|
|
Random rnd(301);
|
||
|
|
std::string str1 = rnd.RandomString(1020);
|
||
|
|
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
||
|
|
ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_,
|
||
|
|
str1.length()));
|
||
|
|
|
||
|
|
std::string str2 = rnd.RandomString(1020);
|
||
|
|
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
||
|
|
// k1 should be demoted to the secondary cache.
|
||
|
|
ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_,
|
||
|
|
str2.length()));
|
||
|
|
|
||
|
|
Cache::Handle* handle;
|
||
|
|
SetFailCreate(true);
|
||
|
|
handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
cache->Release(handle);
|
||
|
|
// This lookup should fail, since k1 creation would have failed
|
||
|
|
handle = cache->Lookup("k1", &LRUSecondaryCacheTest::helper_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_EQ(handle, nullptr);
|
||
|
|
// Since k1 didn't get promoted, k2 should still be in cache
|
||
|
|
handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
cache->Release(handle);
|
||
|
|
|
||
|
|
cache.reset();
|
||
|
|
secondary_cache.reset();
|
||
|
|
}
|
||
|
|
|
||
|
|
void IntegrationFullCapacityTest(bool sec_cache_is_compressed) {
|
||
|
|
LRUSecondaryCacheOptions secondary_cache_opts;
|
||
|
|
|
||
|
|
if (sec_cache_is_compressed) {
|
||
|
|
if (!LZ4_Supported()) {
|
||
|
|
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
|
||
|
|
}
|
||
|
|
|
||
|
|
secondary_cache_opts.capacity = 2048;
|
||
|
|
secondary_cache_opts.num_shard_bits = 0;
|
||
|
|
secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
|
||
|
|
|
||
|
|
std::shared_ptr<SecondaryCache> secondary_cache =
|
||
|
|
NewLRUSecondaryCache(secondary_cache_opts);
|
||
|
|
|
||
|
|
LRUCacheOptions opts(1024, 0, /*_strict_capacity_limit=*/true, 0.5, nullptr,
|
||
|
|
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
|
||
|
|
opts.secondary_cache = secondary_cache;
|
||
|
|
std::shared_ptr<Cache> cache = NewLRUCache(opts);
|
||
|
|
|
||
|
|
Random rnd(301);
|
||
|
|
std::string str1 = rnd.RandomString(1020);
|
||
|
|
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
||
|
|
ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_,
|
||
|
|
str1.length()));
|
||
|
|
std::string str2 = rnd.RandomString(1020);
|
||
|
|
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
||
|
|
// k1 should be demoted to the secondary cache.
|
||
|
|
ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_,
|
||
|
|
str2.length()));
|
||
|
|
|
||
|
|
Cache::Handle* handle;
|
||
|
|
handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
// k1 promotion should fail due to the block cache being at capacity,
|
||
|
|
// but the lookup should still succeed
|
||
|
|
Cache::Handle* handle2;
|
||
|
|
handle2 = cache->Lookup("k1", &LRUSecondaryCacheTest::helper_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_NE(handle2, nullptr);
|
||
|
|
// Since k1 didn't get inserted, k2 should still be in cache
|
||
|
|
cache->Release(handle);
|
||
|
|
cache->Release(handle2);
|
||
|
|
handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
|
||
|
|
test_item_creator, Cache::Priority::LOW, true);
|
||
|
|
ASSERT_NE(handle, nullptr);
|
||
|
|
cache->Release(handle);
|
||
|
|
|
||
|
|
cache.reset();
|
||
|
|
secondary_cache.reset();
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
bool fail_create_;
|
||
|
|
};
|
||
|
|
|
||
|
|
Cache::CacheItemHelper LRUSecondaryCacheTest::helper_(
|
||
|
|
LRUSecondaryCacheTest::SizeCallback, LRUSecondaryCacheTest::SaveToCallback,
|
||
|
|
LRUSecondaryCacheTest::DeletionCallback);
|
||
|
|
|
||
|
|
Cache::CacheItemHelper LRUSecondaryCacheTest::helper_fail_(
|
||
|
|
LRUSecondaryCacheTest::SizeCallback,
|
||
|
|
LRUSecondaryCacheTest::SaveToCallbackFail,
|
||
|
|
LRUSecondaryCacheTest::DeletionCallback);
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicTestWithNoCompression) {
|
||
|
|
BasicTest(false, false);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicTestWithMemoryAllocatorAndNoCompression) {
|
||
|
|
BasicTest(false, true);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicTestWithCompression) {
|
||
|
|
BasicTest(true, false);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicTestWithMemoryAllocatorAndCompression) {
|
||
|
|
BasicTest(true, true);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, FailsTestWithNoCompression) { FailsTest(false); }
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, FailsTestWithCompression) { FailsTest(true); }
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicIntegrationTestWithNoCompression) {
|
||
|
|
BasicIntegrationTest(false);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicIntegrationTestWithCompression) {
|
||
|
|
BasicIntegrationTest(true);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicIntegrationFailTestWithNoCompression) {
|
||
|
|
BasicIntegrationFailTest(false);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, BasicIntegrationFailTestWithCompression) {
|
||
|
|
BasicIntegrationFailTest(true);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, IntegrationSaveFailTestWithNoCompression) {
|
||
|
|
IntegrationSaveFailTest(false);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, IntegrationSaveFailTestWithCompression) {
|
||
|
|
IntegrationSaveFailTest(true);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, IntegrationCreateFailTestWithNoCompression) {
|
||
|
|
IntegrationCreateFailTest(false);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, IntegrationCreateFailTestWithCompression) {
|
||
|
|
IntegrationCreateFailTest(true);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, IntegrationFullCapacityTestWithNoCompression) {
|
||
|
|
IntegrationFullCapacityTest(false);
|
||
|
|
}
|
||
|
|
|
||
|
|
TEST_F(LRUSecondaryCacheTest, IntegrationFullCapacityTestWithCompression) {
|
||
|
|
IntegrationFullCapacityTest(true);
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace ROCKSDB_NAMESPACE
|
||
|
|
|
||
|
|
int main(int argc, char** argv) {
|
||
|
|
::testing::InitGoogleTest(&argc, argv);
|
||
|
|
return RUN_ALL_TESTS();
|
||
|
|
}
|