NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rename stall-related internal stats

Browse Source 
Summary:
Some of these names, like `MEMTABLE_COMPACTION`, did not mean anything. Tried to give them descriptive names.
Closes https://github.com/facebook/rocksdb/pull/2852

Differential Revision: D5782822

Pulled By: ajkr

fbshipit-source-id: f2695c4124af4073da4492d7135bae2411220f3a
main
Andrew Kryczka 7 years ago committed by Facebook Github Bot
parent 0e99323ac2
commit 3cd7ea2e8a
3 changed files with 71 additions and 69 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 17
      db/column_family.cc
  2. 91
      db/internal_stats.cc
  3. 32
      db/internal_stats.h

17
db/column_family.cc
Unescape View File

@ -626,7 +626,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
if (imm()->NumNotFlushed() >= mutable_cf_options.max_write_buffer_number) { if (imm()->NumNotFlushed() >= mutable_cf_options.max_write_buffer_number) {
write_controller_token_ = write_controller->GetStopToken(); write_controller_token_ = write_controller->GetStopToken();
internal_stats_->AddCFStats(InternalStats::MEMTABLE_COMPACTION, 1); internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_STOPS, 1);
ROCKS_LOG_WARN( ROCKS_LOG_WARN(
ioptions_.info_log, ioptions_.info_log,
"[%s] Stopping writes because we have %d immutable memtables " "[%s] Stopping writes because we have %d immutable memtables "
@ -637,10 +637,10 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
vstorage->l0_delay_trigger_count() >= vstorage->l0_delay_trigger_count() >=
mutable_cf_options.level0_stop_writes_trigger) { mutable_cf_options.level0_stop_writes_trigger) {
write_controller_token_ = write_controller->GetStopToken(); write_controller_token_ = write_controller->GetStopToken();
internal_stats_->AddCFStats(InternalStats::LEVEL0_NUM_FILES_TOTAL, 1); internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_STOPS, 1);
if (compaction_picker_->IsLevel0CompactionInProgress()) { if (compaction_picker_->IsLevel0CompactionInProgress()) {
internal_stats_->AddCFStats( internal_stats_->AddCFStats(
InternalStats::LEVEL0_NUM_FILES_WITH_COMPACTION, 1); InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_STOPS, 1);
} }
ROCKS_LOG_WARN(ioptions_.info_log, ROCKS_LOG_WARN(ioptions_.info_log,
"[%s] Stopping writes because we have %d level-0 files", "[%s] Stopping writes because we have %d level-0 files",
@ -651,7 +651,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
mutable_cf_options.hard_pending_compaction_bytes_limit) { mutable_cf_options.hard_pending_compaction_bytes_limit) {
write_controller_token_ = write_controller->GetStopToken(); write_controller_token_ = write_controller->GetStopToken();
internal_stats_->AddCFStats( internal_stats_->AddCFStats(
InternalStats::HARD_PENDING_COMPACTION_BYTES_LIMIT, 1); InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS, 1);
ROCKS_LOG_WARN( ROCKS_LOG_WARN(
ioptions_.info_log, ioptions_.info_log,
"[%s] Stopping writes because of estimated pending compaction " "[%s] Stopping writes because of estimated pending compaction "
@ -664,7 +664,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
SetupDelay(write_controller, compaction_needed_bytes, SetupDelay(write_controller, compaction_needed_bytes,
prev_compaction_needed_bytes_, was_stopped, prev_compaction_needed_bytes_, was_stopped,
mutable_cf_options.disable_auto_compactions); mutable_cf_options.disable_auto_compactions);
internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1); internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_SLOWDOWNS, 1);
ROCKS_LOG_WARN( ROCKS_LOG_WARN(
ioptions_.info_log, ioptions_.info_log,
"[%s] Stalling writes because we have %d immutable memtables " "[%s] Stalling writes because we have %d immutable memtables "
@ -684,10 +684,11 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
SetupDelay(write_controller, compaction_needed_bytes, SetupDelay(write_controller, compaction_needed_bytes,
prev_compaction_needed_bytes_, was_stopped || near_stop, prev_compaction_needed_bytes_, was_stopped || near_stop,
mutable_cf_options.disable_auto_compactions); mutable_cf_options.disable_auto_compactions);
internal_stats_->AddCFStats(InternalStats::LEVEL0_SLOWDOWN_TOTAL, 1); internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_SLOWDOWNS,
1);
if (compaction_picker_->IsLevel0CompactionInProgress()) { if (compaction_picker_->IsLevel0CompactionInProgress()) {
internal_stats_->AddCFStats( internal_stats_->AddCFStats(
InternalStats::LEVEL0_SLOWDOWN_WITH_COMPACTION, 1); InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS, 1);
} }
ROCKS_LOG_WARN(ioptions_.info_log, ROCKS_LOG_WARN(ioptions_.info_log,
"[%s] Stalling writes because we have %d level-0 files " "[%s] Stalling writes because we have %d level-0 files "
@ -714,7 +715,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
prev_compaction_needed_bytes_, was_stopped || near_stop, prev_compaction_needed_bytes_, was_stopped || near_stop,
mutable_cf_options.disable_auto_compactions); mutable_cf_options.disable_auto_compactions);
internal_stats_->AddCFStats( internal_stats_->AddCFStats(
InternalStats::SOFT_PENDING_COMPACTION_BYTES_LIMIT, 1); InternalStats::PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS, 1);
ROCKS_LOG_WARN( ROCKS_LOG_WARN(
ioptions_.info_log, ioptions_.info_log,
"[%s] Stalling writes because of estimated pending compaction " "[%s] Stalling writes because of estimated pending compaction "

91
db/internal_stats.cc
Unescape View File

@ -991,31 +991,30 @@ void InternalStats::DumpCFMapStats(
void InternalStats::DumpCFMapStatsIOStalls( void InternalStats::DumpCFMapStatsIOStalls(
std::map<std::string, std::string>* cf_stats) { std::map<std::string, std::string>* cf_stats) {
(*cf_stats)["io_stalls.level0_slowdown"] = (*cf_stats)["io_stalls.level0_slowdown"] =
std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL]); std::to_string(cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS]);
(*cf_stats)["io_stalls.level0_slowdown_with_compaction"] = (*cf_stats)["io_stalls.level0_slowdown_with_compaction"] =
std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION]); std::to_string(cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS]);
(*cf_stats)["io_stalls.level0_numfiles"] = (*cf_stats)["io_stalls.level0_numfiles"] =
std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_TOTAL]); std::to_string(cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS]);
(*cf_stats)["io_stalls.level0_numfiles_with_compaction"] = (*cf_stats)["io_stalls.level0_numfiles_with_compaction"] =
std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION]); std::to_string(cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_STOPS]);
(*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] = (*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] =
std::to_string(cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT]); std::to_string(cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS]);
(*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] = (*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] =
std::to_string(cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT]); std::to_string(cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS]);
(*cf_stats)["io_stalls.memtable_compaction"] = (*cf_stats)["io_stalls.memtable_compaction"] =
std::to_string(cf_stats_count_[MEMTABLE_COMPACTION]); std::to_string(cf_stats_count_[MEMTABLE_LIMIT_STOPS]);
(*cf_stats)["io_stalls.memtable_slowdown"] = (*cf_stats)["io_stalls.memtable_slowdown"] =
std::to_string(cf_stats_count_[MEMTABLE_SLOWDOWN]); std::to_string(cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS]);
uint64_t total_stop = uint64_t total_stop = cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS] +
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] + cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS] +
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] + cf_stats_count_[MEMTABLE_LIMIT_STOPS];
cf_stats_count_[MEMTABLE_COMPACTION];
uint64_t total_slowdown = uint64_t total_slowdown =
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] + cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS] +
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] + cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS] +
cf_stats_count_[MEMTABLE_SLOWDOWN]; cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS];
(*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop); (*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop);
(*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown); (*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown);
@ -1055,11 +1054,12 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL]; uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL];
// Cumulative summary // Cumulative summary
uint64_t total_stall_count = uint64_t total_stall_count =
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] + cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS] +
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] + cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS] +
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] + cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS] +
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] + cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS] +
cf_stats_count_[MEMTABLE_COMPACTION] + cf_stats_count_[MEMTABLE_SLOWDOWN]; cf_stats_count_[MEMTABLE_LIMIT_STOPS] +
cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS];
// Interval summary // Interval summary
uint64_t interval_flush_ingest = uint64_t interval_flush_ingest =
flush_ingest - cf_stats_snapshot_.ingest_bytes_flush; flush_ingest - cf_stats_snapshot_.ingest_bytes_flush;
@ -1148,31 +1148,32 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
cf_stats_snapshot_.compact_bytes_read = compact_bytes_read; cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
cf_stats_snapshot_.compact_micros = compact_micros; cf_stats_snapshot_.compact_micros = compact_micros;
snprintf(buf, sizeof(buf), "Stalls(count): %" PRIu64 snprintf(buf, sizeof(buf),
" level0_slowdown, " "Stalls(count): %" PRIu64
"%" PRIu64 " level0_slowdown, "
" level0_slowdown_with_compaction, " "%" PRIu64
"%" PRIu64 " level0_slowdown_with_compaction, "
" level0_numfiles, " "%" PRIu64
"%" PRIu64 " level0_numfiles, "
" level0_numfiles_with_compaction, " "%" PRIu64
"%" PRIu64 " level0_numfiles_with_compaction, "
" stop for pending_compaction_bytes, " "%" PRIu64
"%" PRIu64 " stop for pending_compaction_bytes, "
" slowdown for pending_compaction_bytes, " "%" PRIu64
"%" PRIu64 " slowdown for pending_compaction_bytes, "
" memtable_compaction, " "%" PRIu64
"%" PRIu64 " memtable_compaction, "
" memtable_slowdown, " "%" PRIu64
"interval %" PRIu64 " total count\n", " memtable_slowdown, "
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL], "interval %" PRIu64 " total count\n",
cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION], cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS],
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL], cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS],
cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION], cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS],
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT], cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_STOPS],
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT], cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS],
cf_stats_count_[MEMTABLE_COMPACTION], cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS],
cf_stats_count_[MEMTABLE_SLOWDOWN], cf_stats_count_[MEMTABLE_LIMIT_STOPS],
cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS],
total_stall_count - cf_stats_snapshot_.stall_count); total_stall_count - cf_stats_snapshot_.stall_count);
value->append(buf); value->append(buf);

32
db/internal_stats.h
Unescape View File

@ -86,14 +86,14 @@ class InternalStats {
static const std::map<LevelStatType, LevelStat> compaction_level_stats; static const std::map<LevelStatType, LevelStat> compaction_level_stats;
enum InternalCFStatsType { enum InternalCFStatsType {
LEVEL0_SLOWDOWN_TOTAL, L0_FILE_COUNT_LIMIT_SLOWDOWNS,
LEVEL0_SLOWDOWN_WITH_COMPACTION, LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS,
MEMTABLE_COMPACTION, MEMTABLE_LIMIT_STOPS,
MEMTABLE_SLOWDOWN, MEMTABLE_LIMIT_SLOWDOWNS,
LEVEL0_NUM_FILES_TOTAL, L0_FILE_COUNT_LIMIT_STOPS,
LEVEL0_NUM_FILES_WITH_COMPACTION, LOCKED_L0_FILE_COUNT_LIMIT_STOPS,
SOFT_PENDING_COMPACTION_BYTES_LIMIT, PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS,
HARD_PENDING_COMPACTION_BYTES_LIMIT, PENDING_COMPACTION_BYTES_LIMIT_STOPS,
WRITE_STALLS_ENUM_MAX, WRITE_STALLS_ENUM_MAX,
BYTES_FLUSHED, BYTES_FLUSHED,
BYTES_INGESTED_ADD_FILE, BYTES_INGESTED_ADD_FILE,
@ -496,14 +496,14 @@ class InternalStats {
class InternalStats { class InternalStats {
public: public:
enum InternalCFStatsType { enum InternalCFStatsType {
LEVEL0_SLOWDOWN_TOTAL, L0_FILE_COUNT_LIMIT_SLOWDOWNS,
LEVEL0_SLOWDOWN_WITH_COMPACTION, LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS,
MEMTABLE_COMPACTION, MEMTABLE_LIMIT_STOPS,
MEMTABLE_SLOWDOWN, MEMTABLE_LIMIT_SLOWDOWNS,
LEVEL0_NUM_FILES_TOTAL, L0_FILE_COUNT_LIMIT_STOPS,
LEVEL0_NUM_FILES_WITH_COMPACTION, LOCKED_L0_FILE_COUNT_LIMIT_STOPS,
SOFT_PENDING_COMPACTION_BYTES_LIMIT, PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS,
HARD_PENDING_COMPACTION_BYTES_LIMIT, PENDING_COMPACTION_BYTES_LIMIT_STOPS,
WRITE_STALLS_ENUM_MAX, WRITE_STALLS_ENUM_MAX,
BYTES_FLUSHED, BYTES_FLUSHED,
BYTES_INGESTED_ADD_FILE, BYTES_INGESTED_ADD_FILE,

Write Preview
Loading…
Cancel
Save