NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Exporting compaction stats in the form of a map

Browse Source 
Summary:
Currently the compaction stats are printed to stdout. We want to export the compaction stats in a map format so that the upper layer apps (e.g., MySQL) could present
the stats in any format required by the them.
Closes https://github.com/facebook/rocksdb/pull/1477

Differential Revision: D4149836

Pulled By: maysamyabandeh

fbshipit-source-id: b3df19f
main
Maysam Yabandeh 8 years ago committed by Facebook Github Bot
parent 672300f47f
commit 361010d447
8 changed files with 422 additions and 137 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 18
      db/db_impl.cc
  2. 5
      db/db_impl.h
  3. 6
      db/db_test.cc
  4. 25
      db/db_test2.cc
  5. 430
      db/internal_stats.cc
  6. 56
      db/internal_stats.h
  7. 12
      include/rocksdb/db.h
  8. 7
      include/rocksdb/utilities/stackable_db.h

18
db/db_impl.cc
Unescape View File

@ -5292,6 +5292,24 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
return false;
}
bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) {
const DBPropertyInfo* property_info = GetPropertyInfo(property);
value->clear();
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
if (property_info == nullptr) {
return false;
} else if (property_info->handle_map) {
InstrumentedMutexLock l(&mutex_);
return cfd->internal_stats()->GetMapProperty(*property_info, property,
value);
}
// If we reach this point it means that handle_map is not provided for the
// requested property
return false;
}
bool DBImpl::GetIntProperty(ColumnFamilyHandle* column_family,
const Slice& property, uint64_t* value) {
const DBPropertyInfo* property_info = GetPropertyInfo(property);

5
db/db_impl.h
Unescape View File

@ -13,6 +13,7 @@
#include <functional>
#include <limits>
#include <list>
#include <map>
#include <queue>
#include <set>
#include <string>
@ -123,6 +124,10 @@ class DBImpl : public DB {
using DB::GetProperty;
virtual bool GetProperty(ColumnFamilyHandle* column_family,
const Slice& property, std::string* value) override;
using DB::GetMapProperty;
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override;
using DB::GetIntProperty;
virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
const Slice& property, uint64_t* value) override;

6
db/db_test.cc
Unescape View File

@ -2740,6 +2740,12 @@ class ModelDB : public DB {
const Slice& property, uint64_t* value) override {
return false;
}
using DB::GetMapProperty;
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override {
return false;
}
using DB::GetAggregatedIntProperty;
virtual bool GetAggregatedIntProperty(const Slice& property,
uint64_t* value) override {

25
db/db_test2.cc
Unescape View File

@ -2085,8 +2085,26 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
cro.target_level = 2;
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
auto get_stat = [](std::string level_str, LevelStatType type,
std::map<std::string, double> props) {
auto prop_str =
level_str + "." +
InternalStats::compaction_level_stats.at(type).property_name.c_str();
auto prop_item = props.find(prop_str);
return prop_item == props.end() ? 0 : prop_item->second;
};
// Trivial move 2 files to L2
ASSERT_EQ("0,0,2", FilesPerLevel());
// Also test that the stats GetMapProperty API reporting the same result
{
std::map<std::string, double> prop;
ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop));
ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop));
ASSERT_EQ(2, get_stat("L2", LevelStatType::NUM_FILES, prop));
ASSERT_EQ(2, get_stat("Sum", LevelStatType::NUM_FILES, prop));
}
// While the compaction is running, we will create 2 new files that
// can fit in L2, these 2 files will be moved to L2 and overlap with
@ -2113,6 +2131,13 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
// Test that the stats GetMapProperty API reporting 1 file in L2
{
std::map<std::string, double> prop;
ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop));
}
}
TEST_F(DBTest2, ManualCompactionOverlapManualCompaction) {

430
db/internal_stats.cc
Unescape View File

@ -25,62 +25,146 @@
namespace rocksdb {
#ifndef ROCKSDB_LITE
const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats =
{
{LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}},
{LevelStatType::COMPACTED_FILES,
LevelStat{"CompactedFiles", "CompactedFiles"}},
{LevelStatType::SIZE_MB, LevelStat{"SizeMB", "Size(MB}"}},
{LevelStatType::SCORE, LevelStat{"Score", "Score"}},
{LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB}"}},
{LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB}"}},
{LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB}"}},
{LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB}"}},
{LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB}"}},
{LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB}"}},
{LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}},
{LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s}"}},
{LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s}"}},
{LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec}"}},
{LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt}"}},
{LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec}"}},
{LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}},
{LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}},
};
namespace {
const double kMB = 1048576.0;
const double kGB = kMB * 1024;
const double kMicrosInSec = 1000000.0;
void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
snprintf(
buf, len,
"\n** Compaction Stats [%s] **\n"
"Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) "
"Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) "
"Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop\n"
"--------------------------------------------------------------------"
"-----------------------------------------------------------"
"--------------------------------------\n",
cf_name.c_str());
int written_size =
snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str());
auto hdr = [](LevelStatType t) {
return InternalStats::compaction_level_stats.at(t).header_name.c_str();
};
int line_size = snprintf(
buf + written_size, len - written_size,
"Level %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
// Note that we skip COMPACTED_FILES and merge it with Files column
hdr(LevelStatType::NUM_FILES), hdr(LevelStatType::SIZE_MB),
hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB),
hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB),
hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB),
hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP),
hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS),
hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_COUNT),
hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN),
hdr(LevelStatType::KEY_DROP));
written_size += line_size;
snprintf(buf + written_size, len - written_size, "%s\n",
std::string(line_size, '-').c_str());
}
void PrintLevelStats(char* buf, size_t len, const std::string& name,
int num_files, int being_compacted, double total_file_size,
double score, double w_amp,
const InternalStats::CompactionStats& stats) {
void PrepareLevelStats(std::map<LevelStatType, double>* level_stats,
int num_files, int being_compacted,
double total_file_size, double score, double w_amp,
const InternalStats::CompactionStats& stats) {
uint64_t bytes_read =
stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
int64_t bytes_new =
stats.bytes_written - stats.bytes_read_output_level;
double elapsed = (stats.micros + 1) / kMicrosInSec;
std::string num_input_records = NumberToHumanString(stats.num_input_records);
std::string num_dropped_records =
NumberToHumanString(stats.num_dropped_records);
(*level_stats)[LevelStatType::NUM_FILES] = num_files;
(*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted;
(*level_stats)[LevelStatType::SIZE_MB] = total_file_size / kMB;
(*level_stats)[LevelStatType::SCORE] = score;
(*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB;
(*level_stats)[LevelStatType::RN_GB] =
stats.bytes_read_non_output_levels / kGB;
(*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB;
(*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB;
(*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB;
(*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB;
(*level_stats)[LevelStatType::WRITE_AMP] = w_amp;
(*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed;
(*level_stats)[LevelStatType::WRITE_MBPS] =
stats.bytes_written / kMB / elapsed;
(*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec;
(*level_stats)[LevelStatType::COMP_COUNT] = stats.count;
(*level_stats)[LevelStatType::AVG_SEC] =
stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count;
(*level_stats)[LevelStatType::KEY_IN] =
static_cast<double>(stats.num_input_records);
(*level_stats)[LevelStatType::KEY_DROP] =
static_cast<double>(stats.num_dropped_records);
}
void PrintLevelStats(char* buf, size_t len, const std::string& name,
const std::map<LevelStatType, double>& stat_value) {
snprintf(buf, len,
"%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */
"%8.1f " /* Read(GB) */
"%7.1f " /* Rn(GB) */
"%8.1f " /* Rnp1(GB) */
"%9.1f " /* Write(GB) */
"%8.1f " /* Wnew(GB) */
"%9.1f " /* Moved(GB) */
"%5.1f " /* W-Amp */
"%8.1f " /* Rd(MB/s) */
"%8.1f " /* Wr(MB/s) */
"%9.0f " /* Comp(sec) */
"%9d " /* Comp(cnt) */
"%8.3f " /* Avg(sec) */
"%7s " /* KeyIn */
"%6s\n", /* KeyDrop */
"%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */
"%8.1f " /* Read(GB) */
"%7.1f " /* Rn(GB) */
"%8.1f " /* Rnp1(GB) */
"%9.1f " /* Write(GB) */
"%8.1f " /* Wnew(GB) */
"%9.1f " /* Moved(GB) */
"%5.1f " /* W-Amp */
"%8.1f " /* Rd(MB/s) */
"%8.1f " /* Wr(MB/s) */
"%9.0f " /* Comp(sec) */
"%9d " /* Comp(cnt) */
"%8.3f " /* Avg(sec) */
"%7s " /* KeyIn */
"%6s\n", /* KeyDrop */
name.c_str(),
num_files, being_compacted, total_file_size / kMB, score,
bytes_read / kGB, stats.bytes_read_non_output_levels / kGB,
stats.bytes_read_output_level / kGB, stats.bytes_written / kGB,
bytes_new / kGB, stats.bytes_moved / kGB, w_amp,
bytes_read / kMB / elapsed, stats.bytes_written / kMB / elapsed,
stats.micros / kMicrosInSec, stats.count,
stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count,
num_input_records.c_str(), num_dropped_records.c_str());
static_cast<int>(stat_value.at(LevelStatType::NUM_FILES)),
static_cast<int>(stat_value.at(LevelStatType::COMPACTED_FILES)),
stat_value.at(LevelStatType::SIZE_MB),
stat_value.at(LevelStatType::SCORE),
stat_value.at(LevelStatType::READ_GB),
stat_value.at(LevelStatType::RN_GB),
stat_value.at(LevelStatType::RNP1_GB),
stat_value.at(LevelStatType::WRITE_GB),
stat_value.at(LevelStatType::W_NEW_GB),
stat_value.at(LevelStatType::MOVED_GB),
stat_value.at(LevelStatType::WRITE_AMP),
stat_value.at(LevelStatType::READ_MBPS),
stat_value.at(LevelStatType::WRITE_MBPS),
stat_value.at(LevelStatType::COMP_SEC),
static_cast<int>(stat_value.at(LevelStatType::COMP_COUNT)),
stat_value.at(LevelStatType::AVG_SEC),
NumberToHumanString(
static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_IN)))
.c_str(),
NumberToHumanString(static_cast<std::int64_t>(
stat_value.at(LevelStatType::KEY_DROP)))
.c_str());
}
void PrintLevelStats(char* buf, size_t len, const std::string& name,
int num_files, int being_compacted, double total_file_size,
double score, double w_amp,
const InternalStats::CompactionStats& stats) {
std::map<LevelStatType, double> level_stats;
PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size,
score, w_amp, stats);
PrintLevelStats(buf, len, name, level_stats);
}
// Assumes that trailing numbers represent an optional argument. This requires
@ -211,73 +295,91 @@ const std::string DB::Properties::kAggregatedTableProperties =
const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
rocksdb_prefix + aggregated_table_properties_at_level;
const std::unordered_map<std::string,
DBPropertyInfo> InternalStats::ppt_name_to_info = {
{DB::Properties::kNumFilesAtLevelPrefix,
{false, &InternalStats::HandleNumFilesAtLevel, nullptr}},
{DB::Properties::kCompressionRatioAtLevelPrefix,
{false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr}},
{DB::Properties::kLevelStats,
{false, &InternalStats::HandleLevelStats, nullptr}},
{DB::Properties::kStats, {false, &InternalStats::HandleStats, nullptr}},
{DB::Properties::kCFStats, {false, &InternalStats::HandleCFStats, nullptr}},
{DB::Properties::kDBStats, {false, &InternalStats::HandleDBStats, nullptr}},
{DB::Properties::kSSTables,
{false, &InternalStats::HandleSsTables, nullptr}},
{DB::Properties::kAggregatedTableProperties,
{false, &InternalStats::HandleAggregatedTableProperties, nullptr}},
{DB::Properties::kAggregatedTablePropertiesAtLevel,
{false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, nullptr}},
{DB::Properties::kNumImmutableMemTable,
{false, nullptr, &InternalStats::HandleNumImmutableMemTable}},
{DB::Properties::kNumImmutableMemTableFlushed,
{false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed}},
{DB::Properties::kMemTableFlushPending,
{false, nullptr, &InternalStats::HandleMemTableFlushPending}},
{DB::Properties::kCompactionPending,
{false, nullptr, &InternalStats::HandleCompactionPending}},
{DB::Properties::kBackgroundErrors,
{false, nullptr, &InternalStats::HandleBackgroundErrors}},
{DB::Properties::kCurSizeActiveMemTable,
{false, nullptr, &InternalStats::HandleCurSizeActiveMemTable}},
{DB::Properties::kCurSizeAllMemTables,
{false, nullptr, &InternalStats::HandleCurSizeAllMemTables}},
{DB::Properties::kSizeAllMemTables,
{false, nullptr, &InternalStats::HandleSizeAllMemTables}},
{DB::Properties::kNumEntriesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable}},
{DB::Properties::kNumEntriesImmMemTables,
{false, nullptr, &InternalStats::HandleNumEntriesImmMemTables}},
{DB::Properties::kNumDeletesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable}},
{DB::Properties::kNumDeletesImmMemTables,
{false, nullptr, &InternalStats::HandleNumDeletesImmMemTables}},
{DB::Properties::kEstimateNumKeys,
{false, nullptr, &InternalStats::HandleEstimateNumKeys}},
{DB::Properties::kEstimateTableReadersMem,
{true, nullptr, &InternalStats::HandleEstimateTableReadersMem}},
{DB::Properties::kIsFileDeletionsEnabled,
{false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled}},
{DB::Properties::kNumSnapshots,
{false, nullptr, &InternalStats::HandleNumSnapshots}},
{DB::Properties::kOldestSnapshotTime,
{false, nullptr, &InternalStats::HandleOldestSnapshotTime}},
{DB::Properties::kNumLiveVersions,
{false, nullptr, &InternalStats::HandleNumLiveVersions}},
{DB::Properties::kCurrentSuperVersionNumber,
{false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber}},
{DB::Properties::kEstimateLiveDataSize,
{true, nullptr, &InternalStats::HandleEstimateLiveDataSize}},
{DB::Properties::kBaseLevel,
{false, nullptr, &InternalStats::HandleBaseLevel}},
{DB::Properties::kTotalSstFilesSize,
{false, nullptr, &InternalStats::HandleTotalSstFilesSize}},
{DB::Properties::kEstimatePendingCompactionBytes,
{false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes}},
{DB::Properties::kNumRunningFlushes,
{false, nullptr, &InternalStats::HandleNumRunningFlushes}},
{DB::Properties::kNumRunningCompactions,
{false, nullptr, &InternalStats::HandleNumRunningCompactions}},
const std::unordered_map<std::string, DBPropertyInfo>
InternalStats::ppt_name_to_info = {
{DB::Properties::kNumFilesAtLevelPrefix,
{false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr}},
{DB::Properties::kCompressionRatioAtLevelPrefix,
{false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr,
nullptr}},
{DB::Properties::kLevelStats,
{false, &InternalStats::HandleLevelStats, nullptr, nullptr}},
{DB::Properties::kStats,
{false, &InternalStats::HandleStats, nullptr, nullptr}},
{DB::Properties::kCFStats,
{false, &InternalStats::HandleCFStats, nullptr,
&InternalStats::HandleCFMapStats}},
{DB::Properties::kDBStats,
{false, &InternalStats::HandleDBStats, nullptr, nullptr}},
{DB::Properties::kSSTables,
{false, &InternalStats::HandleSsTables, nullptr, nullptr}},
{DB::Properties::kAggregatedTableProperties,
{false, &InternalStats::HandleAggregatedTableProperties, nullptr,
nullptr}},
{DB::Properties::kAggregatedTablePropertiesAtLevel,
{false, &InternalStats::HandleAggregatedTablePropertiesAtLevel,
nullptr, nullptr}},
{DB::Properties::kNumImmutableMemTable,
{false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr}},
{DB::Properties::kNumImmutableMemTableFlushed,
{false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed,
nullptr}},
{DB::Properties::kMemTableFlushPending,
{false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr}},
{DB::Properties::kCompactionPending,
{false, nullptr, &InternalStats::HandleCompactionPending, nullptr}},
{DB::Properties::kBackgroundErrors,
{false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr}},
{DB::Properties::kCurSizeActiveMemTable,
{false, nullptr, &InternalStats::HandleCurSizeActiveMemTable,
nullptr}},
{DB::Properties::kCurSizeAllMemTables,
{false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr}},
{DB::Properties::kSizeAllMemTables,
{false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr}},
{DB::Properties::kNumEntriesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable,
nullptr}},
{DB::Properties::kNumEntriesImmMemTables,
{false, nullptr, &InternalStats::HandleNumEntriesImmMemTables,
nullptr}},
{DB::Properties::kNumDeletesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable,
nullptr}},
{DB::Properties::kNumDeletesImmMemTables,
{false, nullptr, &InternalStats::HandleNumDeletesImmMemTables,
nullptr}},
{DB::Properties::kEstimateNumKeys,
{false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr}},
{DB::Properties::kEstimateTableReadersMem,
{true, nullptr, &InternalStats::HandleEstimateTableReadersMem,
nullptr}},
{DB::Properties::kIsFileDeletionsEnabled,
{false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled,
nullptr}},
{DB::Properties::kNumSnapshots,
{false, nullptr, &InternalStats::HandleNumSnapshots, nullptr}},
{DB::Properties::kOldestSnapshotTime,
{false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr}},
{DB::Properties::kNumLiveVersions,
{false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr}},
{DB::Properties::kCurrentSuperVersionNumber,
{false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber,
nullptr}},
{DB::Properties::kEstimateLiveDataSize,
{true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr}},
{DB::Properties::kBaseLevel,
{false, nullptr, &InternalStats::HandleBaseLevel, nullptr}},
{DB::Properties::kTotalSstFilesSize,
{false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr}},
{DB::Properties::kEstimatePendingCompactionBytes,
{false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
nullptr}},
{DB::Properties::kNumRunningFlushes,
{false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr}},
{DB::Properties::kNumRunningCompactions,
{false, nullptr, &InternalStats::HandleNumRunningCompactions,
nullptr}},
};
const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
@ -298,6 +400,14 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
return (this->*(property_info.handle_string))(value, arg);
}
bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value) {
assert(value != nullptr);
assert(property_info.handle_map != nullptr);
return (this->*(property_info.handle_map))(value);
}
bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info,
uint64_t* value, DBImpl* db) {
assert(value != nullptr);
@ -370,6 +480,11 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) {
return true;
}
bool InternalStats::HandleCFMapStats(std::map<std::string, double>* cf_stats) {
DumpCFMapStats(cf_stats);
return true;
}
bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
DumpCFStats(value);
return true;
@ -716,7 +831,32 @@ void InternalStats::DumpDBStats(std::string* value) {
db_stats_snapshot_.write_stall_micros = write_stall_micros;
}
void InternalStats::DumpCFStats(std::string* value) {
/**
* Dump Compaction Level stats to a map of stat name to value in double.
* The level in stat name is represented with a prefix "Lx" where "x"
* is the level number. A special level "Sum" represents the sum of a stat
* for all levels.
*/
void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
CompactionStats compaction_stats_sum(0);
std::map<int, std::map<LevelStatType, double>> levels_stats;
DumpCFMapStats(&levels_stats, &compaction_stats_sum);
for (auto const& level_ent : levels_stats) {
auto level_str =
level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first);
for (auto const& stat_ent : level_ent.second) {
auto stat_type = stat_ent.first;
auto key_str =
level_str + "." +
InternalStats::compaction_level_stats.at(stat_type).property_name;
(*cf_stats)[key_str] = stat_ent.second;
}
}
}
int InternalStats::DumpCFMapStats(
std::map<int, std::map<LevelStatType, double>>* levels_stats,
CompactionStats* compaction_stats_sum) {
const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
int num_levels_to_check =
@ -724,7 +864,7 @@ void InternalStats::DumpCFStats(std::string* value) {
? vstorage->num_levels() - 1
: 1;
// Compaction scores are sorted base on its value. Restore them to the
// Compaction scores are sorted based on its value. Restore them to the
// level order
std::vector<double> compaction_score(number_levels_, 0);
for (int i = 0; i < num_levels_to_check; ++i) {
@ -741,12 +881,6 @@ void InternalStats::DumpCFStats(std::string* value) {
}
}
char buf[1000];
// Per-ColumnFamily stats
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
value->append(buf);
CompactionStats stats_sum(0);
int total_files = 0;
int total_files_being_compacted = 0;
double total_file_size = 0;
@ -755,40 +889,67 @@ void InternalStats::DumpCFStats(std::string* value) {
total_files += files;
total_files_being_compacted += files_being_compacted[level];
if (comp_stats_[level].micros > 0 || files > 0) {
stats_sum.Add(comp_stats_[level]);
compaction_stats_sum->Add(comp_stats_[level]);
total_file_size += vstorage->NumLevelBytes(level);
double w_amp =
(comp_stats_[level].bytes_read_non_output_levels == 0) ? 0.0
: static_cast<double>(comp_stats_[level].bytes_written) /
comp_stats_[level].bytes_read_non_output_levels;
PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files,
files_being_compacted[level],
static_cast<double>(vstorage->NumLevelBytes(level)),
compaction_score[level],
w_amp, comp_stats_[level]);
value->append(buf);
(comp_stats_[level].bytes_read_non_output_levels == 0)
? 0.0
: static_cast<double>(comp_stats_[level].bytes_written) /
comp_stats_[level].bytes_read_non_output_levels;
std::map<LevelStatType, double> level_stats;
PrepareLevelStats(&level_stats, files, files_being_compacted[level],
static_cast<double>(vstorage->NumLevelBytes(level)),
compaction_score[level], w_amp, comp_stats_[level]);
(*levels_stats)[level] = level_stats;
}
}
uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
uint64_t curr_ingest = flush_ingest + add_file_ingest;
// Cumulative summary
double w_amp = compaction_stats_sum->bytes_written /
static_cast<double>(curr_ingest + 1);
// Stats summary across levels
std::map<LevelStatType, double> sum_stats;
PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted,
total_file_size, 0, w_amp, *compaction_stats_sum);
(*levels_stats)[-1] = sum_stats; // -1 is for the Sum level
return num_levels_to_check;
}
void InternalStats::DumpCFStats(std::string* value) {
char buf[1000];
// Per-ColumnFamily stats
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
value->append(buf);
// Print stats for each level
std::map<int, std::map<LevelStatType, double>> levels_stats;
CompactionStats compaction_stats_sum(0);
int levels = DumpCFMapStats(&levels_stats, &compaction_stats_sum);
for (int l = 0; l < levels; ++l) {
if (levels_stats.find(l) != levels_stats.end()) {
PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]);
value->append(buf);
}
}
// Print sum of level stats
PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]);
value->append(buf);
uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL];
uint64_t ingest_l0_files_addfile =
cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL];
uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL];
// Cumulative summary
double w_amp = stats_sum.bytes_written / static_cast<double>(curr_ingest + 1);
uint64_t total_stall_count =
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] +
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
cf_stats_count_[MEMTABLE_COMPACTION] + cf_stats_count_[MEMTABLE_SLOWDOWN];
// Stats summary across levels
PrintLevelStats(buf, sizeof(buf), "Sum", total_files,
total_files_being_compacted, total_file_size, 0, w_amp,
stats_sum);
value->append(buf);
// Interval summary
uint64_t interval_flush_ingest =
flush_ingest - cf_stats_snapshot_.ingest_bytes_flush;
@ -796,9 +957,10 @@ void InternalStats::DumpCFStats(std::string* value) {
add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile;
uint64_t interval_ingest =
interval_flush_ingest + interval_add_file_inget + 1;
CompactionStats interval_stats(stats_sum);
CompactionStats interval_stats(compaction_stats_sum);
interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
w_amp = interval_stats.bytes_written / static_cast<double>(interval_ingest);
double w_amp =
interval_stats.bytes_written / static_cast<double>(interval_ingest);
PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats);
value->append(buf);
@ -909,7 +1071,7 @@ void InternalStats::DumpCFStats(std::string* value) {
cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
cf_stats_snapshot_.comp_stats = stats_sum;
cf_stats_snapshot_.comp_stats = compaction_stats_sum;
cf_stats_snapshot_.stall_count = total_stall_count;
}

56
db/internal_stats.h
Unescape View File

@ -9,10 +9,11 @@
//
#pragma once
#include "db/version_set.h"
#include <vector>
#include <map>
#include <string>
#include <vector>
#include "db/version_set.h"
class ColumnFamilyData;
@ -41,13 +42,47 @@ struct DBPropertyInfo {
// holding db mutex, which is only supported for int properties.
bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db,
Version* version);
bool (InternalStats::*handle_map)(
std::map<std::string, double>* compaction_stats);
};
extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
#ifndef ROCKSDB_LITE
enum class LevelStatType {
INVALID = 0,
NUM_FILES,
COMPACTED_FILES,
SIZE_MB,
SCORE,
READ_GB,
RN_GB,
RNP1_GB,
WRITE_GB,
W_NEW_GB,
MOVED_GB,
WRITE_AMP,
READ_MBPS,
WRITE_MBPS,
COMP_SEC,
COMP_COUNT,
AVG_SEC,
KEY_IN,
KEY_DROP,
TOTAL // total number of types
};
struct LevelStat {
// This what will be L?.property_name in the flat map returned to the user
std::string property_name;
// This will be what we will print in the header in the cli
std::string header_name;
};
class InternalStats {
public:
static const std::map<LevelStatType, LevelStat> compaction_level_stats;
enum InternalCFStatsType {
LEVEL0_SLOWDOWN_TOTAL,
LEVEL0_SLOWDOWN_WITH_COMPACTION,
@ -221,6 +256,10 @@ class InternalStats {
bool GetStringProperty(const DBPropertyInfo& property_info,
const Slice& property, std::string* value);
bool GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value);
bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
DBImpl* db);
@ -233,6 +272,10 @@ class InternalStats {
private:
void DumpDBStats(std::string* value);
void DumpCFMapStats(std::map<std::string, double>* cf_stats);
int DumpCFMapStats(
std::map<int, std::map<LevelStatType, double>>* level_stats,
CompactionStats* compaction_stats_sum);
void DumpCFStats(std::string* value);
// Per-DB stats
@ -313,6 +356,7 @@ class InternalStats {
bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix);
bool HandleLevelStats(std::string* value, Slice suffix);
bool HandleStats(std::string* value, Slice suffix);
bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
bool HandleCFStats(std::string* value, Slice suffix);
bool HandleDBStats(std::string* value, Slice suffix);
bool HandleSsTables(std::string* value, Slice suffix);
@ -448,6 +492,12 @@ class InternalStats {
return false;
}
bool GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value) {
return false;
}
bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
DBImpl* db) const {
return false;

12
include/rocksdb/db.h
Unescape View File

@ -11,6 +11,7 @@
#include <stdint.h>
#include <stdio.h>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
@ -372,6 +373,10 @@ class DB {
// family stats per-level over db's lifetime ("L<n>"), aggregated over
// db's lifetime ("Sum"), and aggregated over the interval since the
// last retrieval ("Int").
// It could also be used to return the stats in the format of the map.
// In this case there will a pair of string to array of double for
// each level as well as for "Sum". "Int" stats will not be affected
// when this form of stats are retrived.
static const std::string kCFStats;
// "rocksdb.dbstats" - returns a multi-line string with general database
@ -511,6 +516,13 @@ class DB {
virtual bool GetProperty(const Slice& property, std::string* value) {
return GetProperty(DefaultColumnFamily(), property, value);
}
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) = 0;
virtual bool GetMapProperty(const Slice& property,
std::map<std::string, double>* value) {
return GetMapProperty(DefaultColumnFamily(), property, value);
}
// Similar to GetProperty(), but only works for a subset of properties whose
// return value is an integer. Return the value by integer. Supported

7
include/rocksdb/utilities/stackable_db.h
Unescape View File

@ -3,6 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <map>
#include <string>
#include "rocksdb/db.h"
@ -133,11 +134,17 @@ class StackableDB : public DB {
return db_->ReleaseSnapshot(snapshot);
}
using DB::GetMapProperty;
using DB::GetProperty;
virtual bool GetProperty(ColumnFamilyHandle* column_family,
const Slice& property, std::string* value) override {
return db_->GetProperty(column_family, property, value);
}
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override {
return db_->GetMapProperty(column_family, property, value);
}
using DB::GetIntProperty;
virtual bool GetIntProperty(ColumnFamilyHandle* column_family,

Write Preview
Loading…
Cancel
Save