Add block cache related DB properties

Summary:
Add DB properties "rocksdb.block-cache-capacity", "rocksdb.block-cache-usage", "rocksdb.block-cache-pinned-usage" to show block cache usage.
Closes https://github.com/facebook/rocksdb/pull/3734

Differential Revision: D7657180

Pulled By: yiwu-arbug

fbshipit-source-id: dd34a019d5878dab539c51ee82669e97b2b745fd
main
Yi Wu 7 years ago committed by Facebook Github Bot
parent 3cea61392f
commit ad511684b2
  1. 1
      HISTORY.md
  2. 101
      db/db_properties_test.cc
  3. 71
      db/internal_stats.cc
  4. 6
      db/internal_stats.h
  5. 14
      include/rocksdb/db.h

@ -7,6 +7,7 @@
### New Features
* Introduce TTL for level compaction so that all files older than ttl go through the compaction process to get rid of old data.
* TransactionDBOptions::write_policy can be configured to enable WritePrepared 2PC transactions. Read more about them in the wiki.
* Add DB properties "rocksdb.block-cache-capacity", "rocksdb.block-cache-usage", "rocksdb.block-cache-pinned-usage" to show block cache usage.
### Bug Fixes
* Fsync after writing global seq number to the ingestion file in ExternalSstFileIngestionJob.

@ -1436,6 +1436,107 @@ TEST_F(DBPropertiesTest, SstFilesSize) {
ASSERT_TRUE(listener->callback_triggered);
}
TEST_F(DBPropertiesTest, BlockCacheProperties) {
Options options;
uint64_t value;
// Block cache properties are not available for tables other than
// block-based table.
options.table_factory.reset(NewPlainTableFactory());
Reopen(options);
ASSERT_FALSE(
db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
ASSERT_FALSE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
options.table_factory.reset(NewCuckooTableFactory());
Reopen(options);
ASSERT_FALSE(
db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
ASSERT_FALSE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
// Block cache properties are not available if block cache is not used.
BlockBasedTableOptions table_options;
table_options.no_block_cache = true;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options);
ASSERT_FALSE(
db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
ASSERT_FALSE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
// Test with empty block cache.
constexpr size_t kCapacity = 100;
auto block_cache = NewLRUCache(kCapacity, 0 /*num_shard_bits*/);
table_options.block_cache = block_cache;
table_options.no_block_cache = false;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_EQ(kCapacity, value);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
ASSERT_EQ(0, value);
ASSERT_TRUE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
ASSERT_EQ(0, value);
// Insert unpinned item to the cache and check size.
constexpr size_t kSize1 = 50;
block_cache->Insert("item1", nullptr /*value*/, kSize1, nullptr /*deleter*/);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_EQ(kCapacity, value);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
ASSERT_EQ(kSize1, value);
ASSERT_TRUE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
ASSERT_EQ(0, value);
// Insert pinned item to the cache and check size.
constexpr size_t kSize2 = 30;
Cache::Handle* item2 = nullptr;
block_cache->Insert("item2", nullptr /*value*/, kSize2, nullptr /*deleter*/,
&item2);
ASSERT_NE(nullptr, item2);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_EQ(kCapacity, value);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
ASSERT_EQ(kSize1 + kSize2, value);
ASSERT_TRUE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
ASSERT_EQ(kSize2, value);
// Insert another pinned item to make the cache over-sized.
constexpr size_t kSize3 = 80;
Cache::Handle* item3 = nullptr;
block_cache->Insert("item3", nullptr /*value*/, kSize3, nullptr /*deleter*/,
&item3);
ASSERT_NE(nullptr, item2);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_EQ(kCapacity, value);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
// Item 1 is evicted.
ASSERT_EQ(kSize2 + kSize3, value);
ASSERT_TRUE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
ASSERT_EQ(kSize2 + kSize3, value);
// Check size after release.
block_cache->Release(item2);
block_cache->Release(item3);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
ASSERT_EQ(kCapacity, value);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
// item2 will be evicted, while item3 remain in cache after release.
ASSERT_EQ(kSize3, value);
ASSERT_TRUE(
db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
ASSERT_EQ(0, value);
}
#endif // ROCKSDB_LITE
} // namespace rocksdb

@ -18,9 +18,10 @@
#include <string>
#include <utility>
#include <vector>
#include "db/column_family.h"
#include "db/column_family.h"
#include "db/db_impl.h"
#include "table/block_based_table_factory.h"
#include "util/string_util.h"
namespace rocksdb {
@ -245,6 +246,9 @@ static const std::string actual_delayed_write_rate =
"actual-delayed-write-rate";
static const std::string is_write_stopped = "is-write-stopped";
static const std::string estimate_oldest_key_time = "estimate-oldest-key-time";
static const std::string block_cache_capacity = "block-cache-capacity";
static const std::string block_cache_usage = "block-cache-usage";
static const std::string block_cache_pinned_usage = "block-cache-pinned-usage";
const std::string DB::Properties::kNumFilesAtLevelPrefix =
rocksdb_prefix + num_files_at_level_prefix;
@ -322,6 +326,12 @@ const std::string DB::Properties::kIsWriteStopped =
rocksdb_prefix + is_write_stopped;
const std::string DB::Properties::kEstimateOldestKeyTime =
rocksdb_prefix + estimate_oldest_key_time;
const std::string DB::Properties::kBlockCacheCapacity =
rocksdb_prefix + block_cache_capacity;
const std::string DB::Properties::kBlockCacheUsage =
rocksdb_prefix + block_cache_usage;
const std::string DB::Properties::kBlockCachePinnedUsage =
rocksdb_prefix + block_cache_pinned_usage;
const std::unordered_map<std::string, DBPropertyInfo>
InternalStats::ppt_name_to_info = {
@ -425,6 +435,13 @@ const std::unordered_map<std::string, DBPropertyInfo>
{DB::Properties::kEstimateOldestKeyTime,
{false, nullptr, &InternalStats::HandleEstimateOldestKeyTime,
nullptr}},
{DB::Properties::kBlockCacheCapacity,
{false, nullptr, &InternalStats::HandleBlockCacheCapacity, nullptr}},
{DB::Properties::kBlockCacheUsage,
{false, nullptr, &InternalStats::HandleBlockCacheUsage, nullptr}},
{DB::Properties::kBlockCachePinnedUsage,
{false, nullptr, &InternalStats::HandleBlockCachePinnedUsage,
nullptr}},
};
const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
@ -830,6 +847,58 @@ bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,
return *value > 0 && *value < std::numeric_limits<uint64_t>::max();
}
bool InternalStats::HandleBlockCacheStat(Cache** block_cache) {
assert(block_cache != nullptr);
auto* table_factory = cfd_->ioptions()->table_factory;
assert(table_factory != nullptr);
if (BlockBasedTableFactory::kName != table_factory->Name()) {
return false;
}
auto* table_options =
reinterpret_cast<BlockBasedTableOptions*>(table_factory->GetOptions());
if (table_options == nullptr) {
return false;
}
*block_cache = table_options->block_cache.get();
if (table_options->no_block_cache || *block_cache == nullptr) {
return false;
}
return true;
}
bool InternalStats::HandleBlockCacheCapacity(uint64_t* value, DBImpl* /*db*/,
Version* /*version*/) {
Cache* block_cache;
bool ok = HandleBlockCacheStat(&block_cache);
if (!ok) {
return false;
}
*value = static_cast<uint64_t>(block_cache->GetCapacity());
return true;
}
bool InternalStats::HandleBlockCacheUsage(uint64_t* value, DBImpl* /*db*/,
Version* /*version*/) {
Cache* block_cache;
bool ok = HandleBlockCacheStat(&block_cache);
if (!ok) {
return false;
}
*value = static_cast<uint64_t>(block_cache->GetUsage());
return true;
}
bool InternalStats::HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* /*db*/,
Version* /*version*/) {
Cache* block_cache;
bool ok = HandleBlockCacheStat(&block_cache);
if (!ok) {
return false;
}
*value = static_cast<uint64_t>(block_cache->GetPinnedUsage());
return true;
}
void InternalStats::DumpDBStats(std::string* value) {
char buf[1000];
// DB-level stats, only available from default column family

@ -375,6 +375,8 @@ class InternalStats {
void DumpCFStatsNoFileHistogram(std::string* value);
void DumpCFFileHistogram(std::string* value);
bool HandleBlockCacheStat(Cache** block_cache);
// Per-DB stats
std::atomic<uint64_t> db_stats_[INTERNAL_DB_STATS_ENUM_MAX];
// Per-ColumnFamily stats
@ -532,6 +534,10 @@ class InternalStats {
bool HandleIsWriteStopped(uint64_t* value, DBImpl* db, Version* version);
bool HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* db,
Version* version);
bool HandleBlockCacheCapacity(uint64_t* value, DBImpl* db, Version* version);
bool HandleBlockCacheUsage(uint64_t* value, DBImpl* db, Version* version);
bool HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* db,
Version* version);
// Total number of background errors encountered. Every time a flush task
// or compaction task fails, this counter is incremented. The failure can

@ -611,6 +611,17 @@ class DB {
// FIFO compaction with
// compaction_options_fifo.allow_compaction = false.
static const std::string kEstimateOldestKeyTime;
// "rocksdb.block-cache-capacity" - returns block cache capacity.
static const std::string kBlockCacheCapacity;
// "rocksdb.block-cache-usage" - returns the memory size for the entries
// residing in block cache.
static const std::string kBlockCacheUsage;
// "rocksdb.block-cache-pinned-usage" - returns the memory size for the
// entries being pinned.
static const std::string kBlockCachePinnedUsage;
};
#endif /* ROCKSDB_LITE */
@ -663,6 +674,9 @@ class DB {
// "rocksdb.actual-delayed-write-rate"
// "rocksdb.is-write-stopped"
// "rocksdb.estimate-oldest-key-time"
// "rocksdb.block-cache-capacity"
// "rocksdb.block-cache-usage"
// "rocksdb.block-cache-pinned-usage"
virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
const Slice& property, uint64_t* value) = 0;
virtual bool GetIntProperty(const Slice& property, uint64_t* value) {

Loading…
Cancel
Save