Add DB property for fast block cache stats collection (#10832)

Summary:
This new property allows users to trigger the background block cache stats collection mode through the `GetProperty()` and `GetMapProperty()` APIs. The background mode has much lower overhead at the expense of returning stale values in more cases.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10832

Test Plan: updated unit test

Reviewed By: pdillinger

Differential Revision: D40497883

Pulled By: ajkr

fbshipit-source-id: bdcc93402f426463abb2153756aad9e295447343
main
Andrew Kryczka 2 years ago committed by Facebook GitHub Bot
parent 7555243bcf
commit 33ceea9b76
  1. 1
      HISTORY.md
  2. 38
      db/db_block_cache_test.cc
  3. 39
      db/internal_stats.cc
  4. 6
      db/internal_stats.h
  5. 4
      include/rocksdb/db.h

@ -4,6 +4,7 @@
* `DeleteRange()` now supports user-defined timestamp. * `DeleteRange()` now supports user-defined timestamp.
* Provide support for async_io with tailing iterators when ReadOptions.tailing is enabled during scans. * Provide support for async_io with tailing iterators when ReadOptions.tailing is enabled during scans.
* Tiered Storage: allow data moving up from the last level to the penultimate level if the input level is penultimate level or above. * Tiered Storage: allow data moving up from the last level to the penultimate level if the input level is penultimate level or above.
* Added `DB::Properties::kFastBlockCacheEntryStats`, which is similar to `DB::Properties::kBlockCacheEntryStats`, except returns cached (stale) values in more cases to reduce overhead.
* FIFO compaction now supports migrating from a multi-level DB via DB::Open(). During the migration phase, FIFO compaction picker will: * FIFO compaction now supports migrating from a multi-level DB via DB::Open(). During the migration phase, FIFO compaction picker will:
* picks the sst file with the smallest starting key in the bottom-most non-empty level. * picks the sst file with the smallest starting key in the bottom-most non-empty level.
* Note that during the migration phase, the file purge order will only be an approximation of "FIFO" as files in lower-level might sometime contain newer keys than files in upper-level. * Note that during the migration phase, the file purge order will only be an approximation of "FIFO" as files in lower-level might sometime contain newer keys than files in upper-level.

@ -170,13 +170,16 @@ class DBBlockCacheTest : public DBTestBase {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
const std::array<size_t, kNumCacheEntryRoles> GetCacheEntryRoleCountsBg() { const std::array<size_t, kNumCacheEntryRoles> GetCacheEntryRoleCountsBg() {
// Verify in cache entry role stats // Verify in cache entry role stats
ColumnFamilyHandleImpl* cfh = std::array<size_t, kNumCacheEntryRoles> cache_entry_role_counts;
static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily()); std::map<std::string, std::string> values;
InternalStats* internal_stats_ptr = cfh->cfd()->internal_stats(); EXPECT_TRUE(db_->GetMapProperty(DB::Properties::kFastBlockCacheEntryStats,
InternalStats::CacheEntryRoleStats stats; &values));
internal_stats_ptr->TEST_GetCacheEntryRoleStats(&stats, for (size_t i = 0; i < kNumCacheEntryRoles; ++i) {
/*foreground=*/false); auto role = static_cast<CacheEntryRole>(i);
return stats.entry_counts; cache_entry_role_counts[i] =
ParseSizeT(values[BlockCacheEntryStatsMapKeys::EntryCount(role)]);
}
return cache_entry_role_counts;
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
}; };
@ -1500,23 +1503,38 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
dbfull()->DumpStats(); dbfull()->DumpStats();
ASSERT_EQ(scan_count, 1); ASSERT_EQ(scan_count, 1);
env_->MockSleepForSeconds(10000); env_->MockSleepForSeconds(60);
ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kFastBlockCacheEntryStats,
&values));
ASSERT_EQ(scan_count, 1);
ASSERT_TRUE( ASSERT_TRUE(
db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values)); db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values));
ASSERT_EQ(scan_count, 2); ASSERT_EQ(scan_count, 2);
env_->MockSleepForSeconds(10000); env_->MockSleepForSeconds(10000);
ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kFastBlockCacheEntryStats,
&values));
ASSERT_EQ(scan_count, 3);
env_->MockSleepForSeconds(60);
std::string value_str; std::string value_str;
ASSERT_TRUE(db_->GetProperty(DB::Properties::kFastBlockCacheEntryStats,
&value_str));
ASSERT_EQ(scan_count, 3);
ASSERT_TRUE( ASSERT_TRUE(
db_->GetProperty(DB::Properties::kBlockCacheEntryStats, &value_str)); db_->GetProperty(DB::Properties::kBlockCacheEntryStats, &value_str));
ASSERT_EQ(scan_count, 3); ASSERT_EQ(scan_count, 4);
env_->MockSleepForSeconds(10000); env_->MockSleepForSeconds(10000);
ASSERT_TRUE(db_->GetProperty(DB::Properties::kFastBlockCacheEntryStats,
&value_str));
ASSERT_EQ(scan_count, 5);
ASSERT_TRUE(db_->GetProperty(DB::Properties::kCFStats, &value_str)); ASSERT_TRUE(db_->GetProperty(DB::Properties::kCFStats, &value_str));
// To match historical speed, querying this property no longer triggers // To match historical speed, querying this property no longer triggers
// a scan, even if results are old. But periodic dump stats should keep // a scan, even if results are old. But periodic dump stats should keep
// things reasonably updated. // things reasonably updated.
ASSERT_EQ(scan_count, /*unchanged*/ 3); ASSERT_EQ(scan_count, /*unchanged*/ 5);
SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->DisableProcessing();
SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->ClearAllCallBacks();

@ -247,6 +247,8 @@ static const std::string cf_file_histogram = "cf-file-histogram";
static const std::string dbstats = "dbstats"; static const std::string dbstats = "dbstats";
static const std::string levelstats = "levelstats"; static const std::string levelstats = "levelstats";
static const std::string block_cache_entry_stats = "block-cache-entry-stats"; static const std::string block_cache_entry_stats = "block-cache-entry-stats";
static const std::string fast_block_cache_entry_stats =
"fast-block-cache-entry-stats";
static const std::string num_immutable_mem_table = "num-immutable-mem-table"; static const std::string num_immutable_mem_table = "num-immutable-mem-table";
static const std::string num_immutable_mem_table_flushed = static const std::string num_immutable_mem_table_flushed =
"num-immutable-mem-table-flushed"; "num-immutable-mem-table-flushed";
@ -326,6 +328,8 @@ const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats; const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
const std::string DB::Properties::kBlockCacheEntryStats = const std::string DB::Properties::kBlockCacheEntryStats =
rocksdb_prefix + block_cache_entry_stats; rocksdb_prefix + block_cache_entry_stats;
const std::string DB::Properties::kFastBlockCacheEntryStats =
rocksdb_prefix + fast_block_cache_entry_stats;
const std::string DB::Properties::kNumImmutableMemTable = const std::string DB::Properties::kNumImmutableMemTable =
rocksdb_prefix + num_immutable_mem_table; rocksdb_prefix + num_immutable_mem_table;
const std::string DB::Properties::kNumImmutableMemTableFlushed = const std::string DB::Properties::kNumImmutableMemTableFlushed =
@ -446,6 +450,9 @@ const UnorderedMap<std::string, DBPropertyInfo>
{DB::Properties::kBlockCacheEntryStats, {DB::Properties::kBlockCacheEntryStats,
{true, &InternalStats::HandleBlockCacheEntryStats, nullptr, {true, &InternalStats::HandleBlockCacheEntryStats, nullptr,
&InternalStats::HandleBlockCacheEntryStatsMap, nullptr}}, &InternalStats::HandleBlockCacheEntryStatsMap, nullptr}},
{DB::Properties::kFastBlockCacheEntryStats,
{true, &InternalStats::HandleFastBlockCacheEntryStats, nullptr,
&InternalStats::HandleFastBlockCacheEntryStatsMap, nullptr}},
{DB::Properties::kSSTables, {DB::Properties::kSSTables,
{false, &InternalStats::HandleSsTables, nullptr, nullptr, nullptr}}, {false, &InternalStats::HandleSsTables, nullptr, nullptr, nullptr}},
{DB::Properties::kAggregatedTableProperties, {DB::Properties::kAggregatedTableProperties,
@ -739,30 +746,50 @@ void InternalStats::CacheEntryRoleStats::ToMap(
} }
} }
bool InternalStats::HandleBlockCacheEntryStats(std::string* value, bool InternalStats::HandleBlockCacheEntryStatsInternal(std::string* value,
Slice /*suffix*/) { bool fast) {
if (!cache_entry_stats_collector_) { if (!cache_entry_stats_collector_) {
return false; return false;
} }
CollectCacheEntryStats(/*foreground*/ true); CollectCacheEntryStats(!fast /* foreground */);
CacheEntryRoleStats stats; CacheEntryRoleStats stats;
cache_entry_stats_collector_->GetStats(&stats); cache_entry_stats_collector_->GetStats(&stats);
*value = stats.ToString(clock_); *value = stats.ToString(clock_);
return true; return true;
} }
bool InternalStats::HandleBlockCacheEntryStatsMap( bool InternalStats::HandleBlockCacheEntryStatsMapInternal(
std::map<std::string, std::string>* values, Slice /*suffix*/) { std::map<std::string, std::string>* values, bool fast) {
if (!cache_entry_stats_collector_) { if (!cache_entry_stats_collector_) {
return false; return false;
} }
CollectCacheEntryStats(/*foreground*/ true); CollectCacheEntryStats(!fast /* foreground */);
CacheEntryRoleStats stats; CacheEntryRoleStats stats;
cache_entry_stats_collector_->GetStats(&stats); cache_entry_stats_collector_->GetStats(&stats);
stats.ToMap(values, clock_); stats.ToMap(values, clock_);
return true; return true;
} }
bool InternalStats::HandleBlockCacheEntryStats(std::string* value,
Slice /*suffix*/) {
return HandleBlockCacheEntryStatsInternal(value, false /* fast */);
}
bool InternalStats::HandleBlockCacheEntryStatsMap(
std::map<std::string, std::string>* values, Slice /*suffix*/) {
return HandleBlockCacheEntryStatsMapInternal(values, false /* fast */);
}
bool InternalStats::HandleFastBlockCacheEntryStats(std::string* value,
Slice /*suffix*/) {
return HandleBlockCacheEntryStatsInternal(value, true /* fast */);
}
bool InternalStats::HandleFastBlockCacheEntryStatsMap(
std::map<std::string, std::string>* values, Slice /*suffix*/) {
return HandleBlockCacheEntryStatsMapInternal(values, true /* fast */);
}
bool InternalStats::HandleLiveSstFilesSizeAtTemperature(std::string* value, bool InternalStats::HandleLiveSstFilesSizeAtTemperature(std::string* value,
Slice suffix) { Slice suffix) {
uint64_t temperature; uint64_t temperature;

@ -793,9 +793,15 @@ class InternalStats {
bool HandleBlockCacheUsage(uint64_t* value, DBImpl* db, Version* version); bool HandleBlockCacheUsage(uint64_t* value, DBImpl* db, Version* version);
bool HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* db, bool HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* db,
Version* version); Version* version);
bool HandleBlockCacheEntryStatsInternal(std::string* value, bool fast);
bool HandleBlockCacheEntryStatsMapInternal(
std::map<std::string, std::string>* values, bool fast);
bool HandleBlockCacheEntryStats(std::string* value, Slice suffix); bool HandleBlockCacheEntryStats(std::string* value, Slice suffix);
bool HandleBlockCacheEntryStatsMap(std::map<std::string, std::string>* values, bool HandleBlockCacheEntryStatsMap(std::map<std::string, std::string>* values,
Slice suffix); Slice suffix);
bool HandleFastBlockCacheEntryStats(std::string* value, Slice suffix);
bool HandleFastBlockCacheEntryStatsMap(
std::map<std::string, std::string>* values, Slice suffix);
bool HandleLiveSstFilesSizeAtTemperature(std::string* value, Slice suffix); bool HandleLiveSstFilesSizeAtTemperature(std::string* value, Slice suffix);
bool HandleNumBlobFiles(uint64_t* value, DBImpl* db, Version* version); bool HandleNumBlobFiles(uint64_t* value, DBImpl* db, Version* version);
bool HandleBlobStats(std::string* value, Slice suffix); bool HandleBlobStats(std::string* value, Slice suffix);

@ -911,6 +911,10 @@ class DB {
// available in the map form. // available in the map form.
static const std::string kBlockCacheEntryStats; static const std::string kBlockCacheEntryStats;
// "rocksdb.fast-block-cache-entry-stats" - same as above, but returns
// stale values more frequently to reduce overhead and latency.
static const std::string kFastBlockCacheEntryStats;
// "rocksdb.num-immutable-mem-table" - returns number of immutable // "rocksdb.num-immutable-mem-table" - returns number of immutable
// memtables that have not yet been flushed. // memtables that have not yet been flushed.
static const std::string kNumImmutableMemTable; static const std::string kNumImmutableMemTable;

Loading…
Cancel
Save