NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add PerfContext counters for CompressedSecondaryCache (#10650)

Browse Source 
Summary:
Add PerfContext counters for CompressedSecondaryCache.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10650

Test Plan: Unit Tests.

Reviewed By: anand1976

Differential Revision: D39354712

Pulled By: gitbw95

fbshipit-source-id: 1b90d3df99d08ddecd351edfd48d1e3723fdbc15
main
gitbw95 2 years ago committed by Facebook GitHub Bot
parent 3d67d79154
commit 0148c4934d
6 changed files with 135 additions and 7 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      HISTORY.md
  2. 7
      cache/compressed_secondary_cache.cc
  3. 71
      cache/compressed_secondary_cache_test.cc
  4. 5
      cache/lru_cache.cc
  5. 13
      include/rocksdb/perf_context.h
  6. 45
      monitoring/perf_context.cc

1
HISTORY.md
Unescape View File

@ -25,6 +25,7 @@
### New Features
* RocksDB does internal auto prefetching if it notices 2 sequential reads if readahead_size is not specified. New option `num_file_reads_for_auto_readahead` is added in BlockBasedTableOptions which indicates after how many sequential reads internal auto prefetching should be start (default is 2).
* Added new perf context counters `block_cache_standalone_handle_count`, `block_cache_real_handle_count`,`compressed_sec_cache_insert_real_count`, `compressed_sec_cache_insert_dummy_count`, `compressed_sec_cache_uncompressed_bytes`, and `compressed_sec_cache_compressed_bytes`.
### Performance Improvements
* Iterator performance is improved for `DeleteRange()` users. Internally, iterator will skip to the end of a range tombstone when possible, instead of looping through each key and check individually if a key is range deleted.

7
cache/compressed_secondary_cache.cc vendored
Unescape View File

@ -10,6 +10,7 @@
#include <memory>
#include "memory/memory_allocator.h"
#include "monitoring/perf_context_imp.h"
#include "util/compression.h"
#include "util/string_util.h"
@ -101,9 +102,11 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
Cache::Handle* lru_handle = cache_->Lookup(key);
if (lru_handle == nullptr) {
PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1);
// Insert a dummy handle if the handle is evicted for the first time.
return cache_->Insert(key, /*value=*/nullptr, /*charge=*/0,
DeletionCallback);
} else {
cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
}
@ -120,6 +123,7 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
std::string compressed_val;
if (cache_options_.compression_type != kNoCompression) {
PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, size);
CompressionOptions compression_opts;
CompressionContext compression_context(cache_options_.compression_type);
uint64_t sample_for_compression{0};
@ -137,12 +141,13 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
val = Slice(compressed_val);
size = compressed_val.size();
PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, size);
ptr = AllocateBlock(size, cache_options_.memory_allocator.get());
memcpy(ptr.get(), compressed_val.data(), size);
}
CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
PERF_COUNTER_ADD(compressed_sec_cache_insert_real_count, 1);
return cache_->Insert(key, buf, size, DeletionCallback);
}

71
cache/compressed_secondary_cache_test.cc vendored
Unescape View File

@ -84,7 +84,9 @@ class CompressedSecondaryCacheTest : public testing::Test {
void SetFailCreate(bool fail) { fail_create_ = fail; }
void BasicTestHelper(std::shared_ptr<SecondaryCache> sec_cache) {
void BasicTestHelper(std::shared_ptr<SecondaryCache> sec_cache,
bool sec_cache_is_compressed) {
get_perf_context()->Reset();
bool is_in_sec_cache{true};
// Lookup an non-existent key.
std::unique_ptr<SecondaryCacheResultHandle> handle0 = sec_cache->Lookup(
@ -98,18 +100,31 @@ class CompressedSecondaryCacheTest : public testing::Test {
// A dummy handle is inserted if the item is inserted for the first time.
ASSERT_OK(sec_cache->Insert("k1", &item1,
&CompressedSecondaryCacheTest::helper_));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
std::unique_ptr<SecondaryCacheResultHandle> handle1_1 = sec_cache->Lookup(
"k1", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
ASSERT_EQ(handle1_1, nullptr);
// Insert and Lookup the item k1 for the second time.
// Insert and Lookup the item k1 for the second time and advise erasing it.
ASSERT_OK(sec_cache->Insert("k1", &item1,
&CompressedSecondaryCacheTest::helper_));
std::unique_ptr<SecondaryCacheResultHandle> handle1_2 = sec_cache->Lookup(
"k1", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache);
ASSERT_NE(handle1_2, nullptr);
ASSERT_FALSE(is_in_sec_cache);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1);
if (sec_cache_is_compressed) {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
1000);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
1007);
} else {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
}
std::unique_ptr<TestItem> val1 =
std::unique_ptr<TestItem>(static_cast<TestItem*>(handle1_2->Value()));
@ -126,15 +141,26 @@ class CompressedSecondaryCacheTest : public testing::Test {
TestItem item2(str2.data(), str2.length());
ASSERT_OK(sec_cache->Insert("k2", &item2,
&CompressedSecondaryCacheTest::helper_));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2);
std::unique_ptr<SecondaryCacheResultHandle> handle2_1 = sec_cache->Lookup(
"k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
ASSERT_EQ(handle2_1, nullptr);
ASSERT_OK(sec_cache->Insert("k2", &item2,
&CompressedSecondaryCacheTest::helper_));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2);
if (sec_cache_is_compressed) {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
2000);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
2014);
} else {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
}
std::unique_ptr<SecondaryCacheResultHandle> handle2_2 = sec_cache->Lookup(
"k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
ASSERT_EQ(handle2_1, nullptr);
ASSERT_NE(handle2_2, nullptr);
std::unique_ptr<TestItem> val2 =
std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2_2->Value()));
ASSERT_NE(val2, nullptr);
@ -156,6 +182,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
if (!LZ4_Supported()) {
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
opts.compression_type = CompressionType::kNoCompression;
sec_cache_is_compressed = false;
}
} else {
opts.compression_type = CompressionType::kNoCompression;
@ -177,7 +204,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
std::shared_ptr<SecondaryCache> sec_cache =
NewCompressedSecondaryCache(opts);
BasicTestHelper(sec_cache);
BasicTestHelper(sec_cache, sec_cache_is_compressed);
}
void FailsTest(bool sec_cache_is_compressed) {
@ -262,6 +289,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
if (!LZ4_Supported()) {
ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
sec_cache_is_compressed = false;
}
} else {
secondary_cache_opts.compression_type = CompressionType::kNoCompression;
@ -280,6 +308,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
std::shared_ptr<Cache> cache = NewLRUCache(lru_cache_opts);
std::shared_ptr<Statistics> stats = CreateDBStatistics();
get_perf_context()->Reset();
Random rnd(301);
std::string str1 = rnd.RandomString(1001);
TestItem* item1_1 = new TestItem(str1.data(), str1.length());
@ -292,6 +321,9 @@ class CompressedSecondaryCacheTest : public testing::Test {
// k1's dummy item.
ASSERT_OK(cache->Insert(
"k2", item2_1, &CompressedSecondaryCacheTest::helper_, str2.length()));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
std::string str3 = rnd.RandomString(1024);
TestItem* item3_1 = new TestItem(str3.data(), str3.length());
@ -299,24 +331,46 @@ class CompressedSecondaryCacheTest : public testing::Test {
// k1's dummy item and k2's dummy item.
ASSERT_OK(cache->Insert(
"k3", item3_1, &CompressedSecondaryCacheTest::helper_, str3.length()));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2);
// After this Insert, primary cache contains k1 and secondary cache contains
// k1's dummy item, k2's dummy item, and k3's dummy item.
TestItem* item1_2 = new TestItem(str1.data(), str1.length());
ASSERT_OK(cache->Insert(
"k1", item1_2, &CompressedSecondaryCacheTest::helper_, str1.length()));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3);
// After this Insert, primary cache contains k2 and secondary cache contains
// k1's item, k2's dummy item, and k3's dummy item.
TestItem* item2_2 = new TestItem(str2.data(), str2.length());
ASSERT_OK(cache->Insert(
"k2", item2_2, &CompressedSecondaryCacheTest::helper_, str2.length()));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1);
if (sec_cache_is_compressed) {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
str1.length());
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
1008);
} else {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
}
// After this Insert, primary cache contains k3 and secondary cache contains
// k1's item and k2's item.
TestItem* item3_2 = new TestItem(str3.data(), str3.length());
ASSERT_OK(cache->Insert(
"k3", item3_2, &CompressedSecondaryCacheTest::helper_, str3.length()));
ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2);
if (sec_cache_is_compressed) {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
str1.length() + str2.length());
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
2027);
} else {
ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
}
Cache::Handle* handle;
handle = cache->Lookup("k3", &CompressedSecondaryCacheTest::helper_,
@ -340,6 +394,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
test_item_creator, Cache::Priority::LOW, true,
stats.get());
ASSERT_NE(handle, nullptr);
ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1);
TestItem* val1_1 = static_cast<TestItem*>(cache->Value(handle));
ASSERT_NE(val1_1, nullptr);
ASSERT_EQ(memcmp(val1_1->Buf(), str1.data(), str1.size()), 0);
@ -351,6 +406,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
test_item_creator, Cache::Priority::LOW, true,
stats.get());
ASSERT_NE(handle, nullptr);
ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1);
cache->Release(handle);
// k2 is still in secondary cache.
@ -358,6 +414,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
test_item_creator, Cache::Priority::LOW, true,
stats.get());
ASSERT_NE(handle, nullptr);
ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 2);
cache->Release(handle);
cache.reset();
@ -770,11 +827,12 @@ TEST_F(CompressedSecondaryCacheTest, BasicTestFromStringWithNoCompression) {
Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri,
&sec_cache);
EXPECT_OK(s);
BasicTestHelper(sec_cache);
BasicTestHelper(sec_cache, /*sec_cache_is_compressed=*/false);
}
TEST_F(CompressedSecondaryCacheTest, BasicTestFromStringWithCompression) {
std::string sec_cache_uri;
bool sec_cache_is_compressed{true};
if (LZ4_Supported()) {
sec_cache_uri =
"compressed_secondary_cache://"
@ -785,13 +843,14 @@ TEST_F(CompressedSecondaryCacheTest, BasicTestFromStringWithCompression) {
sec_cache_uri =
"compressed_secondary_cache://"
"capacity=2048;num_shard_bits=0;compression_type=kNoCompression";
sec_cache_is_compressed = false;
}
std::shared_ptr<SecondaryCache> sec_cache;
Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri,
&sec_cache);
EXPECT_OK(s);
BasicTestHelper(sec_cache);
BasicTestHelper(sec_cache, sec_cache_is_compressed);
}
#endif // ROCKSDB_LITE

5
cache/lru_cache.cc vendored
Unescape View File

@ -458,6 +458,8 @@ void LRUCacheShard::Promote(LRUHandle* e) {
e->Unref();
e->Free();
e = nullptr;
} else {
PERF_COUNTER_ADD(block_cache_standalone_handle_count, 1);
}
// Insert a dummy handle into the primary cache. This dummy handle is
@ -477,6 +479,9 @@ void LRUCacheShard::Promote(LRUHandle* e) {
// and the caller will most likely just read it from disk if we erase it
// here.
s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
if (s.ok()) {
PERF_COUNTER_ADD(block_cache_real_handle_count, 1);
}
}
if (!s.ok()) {

13
include/rocksdb/perf_context.h
Unescape View File

@ -69,6 +69,11 @@ struct PerfContext {
uint64_t block_read_byte; // total number of bytes from block reads
uint64_t block_read_time; // total nanos spent on block reads
uint64_t block_cache_index_hit_count; // total number of index block hits
// total number of standalone handles lookup from secondary cache
uint64_t block_cache_standalone_handle_count;
// total number of real handles lookup from secondary cache that are inserted
// into primary cache
uint64_t block_cache_real_handle_count;
uint64_t index_block_read_count; // total number of index block reads
uint64_t block_cache_filter_hit_count; // total number of filter block hits
uint64_t filter_block_read_count; // total number of filter block reads
@ -76,6 +81,14 @@ struct PerfContext {
// dictionary block reads
uint64_t secondary_cache_hit_count; // total number of secondary cache hits
// total number of real handles inserted into secondary cache
uint64_t compressed_sec_cache_insert_real_count;
// total number of dummy handles inserted into secondary cache
uint64_t compressed_sec_cache_insert_dummy_count;
// bytes for vals before compression in secondary cache
uint64_t compressed_sec_cache_uncompressed_bytes;
// bytes for vals after compression in secondary cache
uint64_t compressed_sec_cache_compressed_bytes;
uint64_t block_checksum_time; // total nanos spent on block checksum
uint64_t block_decompress_time; // total nanos spent on block decompression

45
monitoring/perf_context.cc
Unescape View File

@ -37,11 +37,22 @@ PerfContext::PerfContext(const PerfContext& other) {
block_read_byte = other.block_read_byte;
block_read_time = other.block_read_time;
block_cache_index_hit_count = other.block_cache_index_hit_count;
block_cache_standalone_handle_count =
other.block_cache_standalone_handle_count;
block_cache_real_handle_count = other.block_cache_real_handle_count;
index_block_read_count = other.index_block_read_count;
block_cache_filter_hit_count = other.block_cache_filter_hit_count;
filter_block_read_count = other.filter_block_read_count;
compression_dict_block_read_count = other.compression_dict_block_read_count;
secondary_cache_hit_count = other.secondary_cache_hit_count;
compressed_sec_cache_insert_real_count =
other.compressed_sec_cache_insert_real_count;
compressed_sec_cache_insert_dummy_count =
other.compressed_sec_cache_insert_dummy_count;
compressed_sec_cache_uncompressed_bytes =
other.compressed_sec_cache_uncompressed_bytes;
compressed_sec_cache_compressed_bytes =
other.compressed_sec_cache_compressed_bytes;
block_checksum_time = other.block_checksum_time;
block_decompress_time = other.block_decompress_time;
get_read_bytes = other.get_read_bytes;
@ -145,11 +156,22 @@ PerfContext::PerfContext(PerfContext&& other) noexcept {
block_read_byte = other.block_read_byte;
block_read_time = other.block_read_time;
block_cache_index_hit_count = other.block_cache_index_hit_count;
block_cache_standalone_handle_count =
other.block_cache_standalone_handle_count;
block_cache_real_handle_count = other.block_cache_real_handle_count;
index_block_read_count = other.index_block_read_count;
block_cache_filter_hit_count = other.block_cache_filter_hit_count;
filter_block_read_count = other.filter_block_read_count;
compression_dict_block_read_count = other.compression_dict_block_read_count;
secondary_cache_hit_count = other.secondary_cache_hit_count;
compressed_sec_cache_insert_real_count =
other.compressed_sec_cache_insert_real_count;
compressed_sec_cache_insert_dummy_count =
other.compressed_sec_cache_insert_dummy_count;
compressed_sec_cache_uncompressed_bytes =
other.compressed_sec_cache_uncompressed_bytes;
compressed_sec_cache_compressed_bytes =
other.compressed_sec_cache_compressed_bytes;
block_checksum_time = other.block_checksum_time;
block_decompress_time = other.block_decompress_time;
get_read_bytes = other.get_read_bytes;
@ -255,11 +277,22 @@ PerfContext& PerfContext::operator=(const PerfContext& other) {
block_read_byte = other.block_read_byte;
block_read_time = other.block_read_time;
block_cache_index_hit_count = other.block_cache_index_hit_count;
block_cache_standalone_handle_count =
other.block_cache_standalone_handle_count;
block_cache_real_handle_count = other.block_cache_real_handle_count;
index_block_read_count = other.index_block_read_count;
block_cache_filter_hit_count = other.block_cache_filter_hit_count;
filter_block_read_count = other.filter_block_read_count;
compression_dict_block_read_count = other.compression_dict_block_read_count;
secondary_cache_hit_count = other.secondary_cache_hit_count;
compressed_sec_cache_insert_real_count =
other.compressed_sec_cache_insert_real_count;
compressed_sec_cache_insert_dummy_count =
other.compressed_sec_cache_insert_dummy_count;
compressed_sec_cache_uncompressed_bytes =
other.compressed_sec_cache_uncompressed_bytes;
compressed_sec_cache_compressed_bytes =
other.compressed_sec_cache_compressed_bytes;
block_checksum_time = other.block_checksum_time;
block_decompress_time = other.block_decompress_time;
get_read_bytes = other.get_read_bytes;
@ -362,11 +395,17 @@ void PerfContext::Reset() {
block_read_byte = 0;
block_read_time = 0;
block_cache_index_hit_count = 0;
block_cache_standalone_handle_count = 0;
block_cache_real_handle_count = 0;
index_block_read_count = 0;
block_cache_filter_hit_count = 0;
filter_block_read_count = 0;
compression_dict_block_read_count = 0;
secondary_cache_hit_count = 0;
compressed_sec_cache_insert_real_count = 0;
compressed_sec_cache_insert_dummy_count = 0;
compressed_sec_cache_uncompressed_bytes = 0;
compressed_sec_cache_compressed_bytes = 0;
block_checksum_time = 0;
block_decompress_time = 0;
get_read_bytes = 0;
@ -493,11 +532,17 @@ std::string PerfContext::ToString(bool exclude_zero_counters) const {
PERF_CONTEXT_OUTPUT(block_read_byte);
PERF_CONTEXT_OUTPUT(block_read_time);
PERF_CONTEXT_OUTPUT(block_cache_index_hit_count);
PERF_CONTEXT_OUTPUT(block_cache_standalone_handle_count);
PERF_CONTEXT_OUTPUT(block_cache_real_handle_count);
PERF_CONTEXT_OUTPUT(index_block_read_count);
PERF_CONTEXT_OUTPUT(block_cache_filter_hit_count);
PERF_CONTEXT_OUTPUT(filter_block_read_count);
PERF_CONTEXT_OUTPUT(compression_dict_block_read_count);
PERF_CONTEXT_OUTPUT(secondary_cache_hit_count);
PERF_CONTEXT_OUTPUT(compressed_sec_cache_insert_real_count);
PERF_CONTEXT_OUTPUT(compressed_sec_cache_insert_dummy_count);
PERF_CONTEXT_OUTPUT(compressed_sec_cache_uncompressed_bytes);
PERF_CONTEXT_OUTPUT(compressed_sec_cache_compressed_bytes);
PERF_CONTEXT_OUTPUT(block_checksum_time);
PERF_CONTEXT_OUTPUT(block_decompress_time);
PERF_CONTEXT_OUTPUT(get_read_bytes);

Write Preview
Loading…
Cancel
Save