From 7b78d623f7205722e133376f51893989cb94d37a Mon Sep 17 00:00:00 2001 From: sdong Date: Thu, 21 Apr 2016 16:58:38 -0700 Subject: [PATCH] Shouldn't report default column family's compaction stats as DB compaction stats Summary: Now we collect compaction stats per column family, but report default colum family's stat as compaction stats for DB. Fix it by reporting compaction stats per column family instead. Test Plan: Run db_bench with --num_column_families=4 and see the number fixed. Reviewers: IslamAbdelRahman, yhchiang Reviewed By: yhchiang Subscribers: leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D57063 --- db/internal_stats.cc | 87 ++++++++++++++++++++++++-------------------- db/internal_stats.h | 18 +++++---- 2 files changed, 58 insertions(+), 47 deletions(-) diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 453cb813a..093663b79 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -608,9 +608,6 @@ void InternalStats::DumpDBStats(std::string* value) { uint64_t wal_synced = GetDBStats(InternalStats::WAL_FILE_SYNCED); uint64_t write_with_wal = GetDBStats(InternalStats::WRITE_WITH_WAL); uint64_t write_stall_micros = GetDBStats(InternalStats::WRITE_STALL_MICROS); - uint64_t compact_bytes_read = 0; - uint64_t compact_bytes_write = 0; - uint64_t compact_micros = 0; const int kHumanMicrosLen = 32; char human_micros[kHumanMicrosLen]; @@ -642,20 +639,6 @@ void InternalStats::DumpDBStats(std::string* value) { write_with_wal / static_cast(wal_synced + 1), wal_bytes / kGB, wal_bytes / kMB / seconds_up); value->append(buf); - // Compact - for (int level = 0; level < number_levels_; level++) { - compact_bytes_read += comp_stats_[level].bytes_read_output_level + - comp_stats_[level].bytes_read_non_output_levels; - compact_bytes_write += comp_stats_[level].bytes_written; - compact_micros += comp_stats_[level].micros; - } - snprintf(buf, sizeof(buf), - "Cumulative compaction: %.2f GB write, %.2f MB/s write, " - "%.2f GB read, %.2f MB/s read, %.1f seconds\n", - compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up, - compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up, - compact_micros / kMicrosInSec); - value->append(buf); // Stall AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true); snprintf(buf, sizeof(buf), @@ -700,25 +683,6 @@ void InternalStats::DumpDBStats(std::string* value) { interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001)); value->append(buf); - // Compaction - uint64_t interval_compact_bytes_write = - compact_bytes_write - db_stats_snapshot_.compact_bytes_write; - uint64_t interval_compact_bytes_read = - compact_bytes_read - db_stats_snapshot_.compact_bytes_read; - uint64_t interval_compact_micros = - compact_micros - db_stats_snapshot_.compact_micros; - - snprintf( - buf, sizeof(buf), - "Interval compaction: %.2f GB write, %.2f MB/s write, " - "%.2f GB read, %.2f MB/s read, %.1f seconds\n", - interval_compact_bytes_write / kGB, - interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001), - interval_compact_bytes_read / kGB, - interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001), - interval_compact_micros / kMicrosInSec); - value->append(buf); - // Stall AppendHumanMicros( write_stall_micros - db_stats_snapshot_.write_stall_micros, @@ -750,9 +714,6 @@ void InternalStats::DumpDBStats(std::string* value) { db_stats_snapshot_.wal_synced = wal_synced; db_stats_snapshot_.write_with_wal = write_with_wal; db_stats_snapshot_.write_stall_micros = write_stall_micros; - db_stats_snapshot_.compact_bytes_write = compact_bytes_write; - db_stats_snapshot_.compact_bytes_read = compact_bytes_read; - db_stats_snapshot_.compact_micros = compact_micros; } void InternalStats::DumpCFStats(std::string* value) { @@ -808,6 +769,7 @@ void InternalStats::DumpCFStats(std::string* value) { value->append(buf); } } + uint64_t curr_ingest = cf_stats_value_[BYTES_FLUSHED]; // Cumulative summary double w_amp = stats_sum.bytes_written / static_cast(curr_ingest + 1); @@ -831,11 +793,58 @@ void InternalStats::DumpCFStats(std::string* value) { PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats); value->append(buf); + double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec; + double interval_seconds_up = seconds_up - cf_stats_snapshot_.seconds_up; + snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n", + seconds_up, interval_seconds_up); + value->append(buf); + snprintf(buf, sizeof(buf), "Flush(GB): cumulative %.3f, interval %.3f\n", curr_ingest / kGB, interval_ingest / kGB); value->append(buf); + // Compact + uint64_t compact_bytes_read = 0; + uint64_t compact_bytes_write = 0; + uint64_t compact_micros = 0; + for (int level = 0; level < number_levels_; level++) { + compact_bytes_read += comp_stats_[level].bytes_read_output_level + + comp_stats_[level].bytes_read_non_output_levels; + compact_bytes_write += comp_stats_[level].bytes_written; + compact_micros += comp_stats_[level].micros; + } + + snprintf(buf, sizeof(buf), + "Cumulative compaction: %.2f GB write, %.2f MB/s write, " + "%.2f GB read, %.2f MB/s read, %.1f seconds\n", + compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up, + compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up, + compact_micros / kMicrosInSec); + value->append(buf); + + // Compaction interval + uint64_t interval_compact_bytes_write = + compact_bytes_write - cf_stats_snapshot_.compact_bytes_write; + uint64_t interval_compact_bytes_read = + compact_bytes_read - cf_stats_snapshot_.compact_bytes_read; + uint64_t interval_compact_micros = + compact_micros - cf_stats_snapshot_.compact_micros; + + snprintf( + buf, sizeof(buf), + "Interval compaction: %.2f GB write, %.2f MB/s write, " + "%.2f GB read, %.2f MB/s read, %.1f seconds\n", + interval_compact_bytes_write / kGB, + interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001), + interval_compact_bytes_read / kGB, + interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001), + interval_compact_micros / kMicrosInSec); + value->append(buf); + cf_stats_snapshot_.compact_bytes_write = compact_bytes_write; + cf_stats_snapshot_.compact_bytes_read = compact_bytes_read; + cf_stats_snapshot_.compact_micros = compact_micros; + snprintf(buf, sizeof(buf), "Stalls(count): %" PRIu64 " level0_slowdown, " "%" PRIu64 diff --git a/db/internal_stats.h b/db/internal_stats.h index 9aad82e7d..fedafe232 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -246,11 +246,20 @@ class InternalStats { CompactionStats comp_stats; uint64_t ingest_bytes; // Bytes written to L0 uint64_t stall_count; // Stall count + // Stats from compaction jobs - bytes written, bytes read, duration. + uint64_t compact_bytes_write; + uint64_t compact_bytes_read; + uint64_t compact_micros; + double seconds_up; CFStatsSnapshot() : comp_stats(0), ingest_bytes(0), - stall_count(0) {} + stall_count(0), + compact_bytes_write(0), + compact_bytes_read(0), + compact_micros(0), + seconds_up(0) {} } cf_stats_snapshot_; struct DBStatsSnapshot { @@ -263,10 +272,6 @@ class InternalStats { // another thread. uint64_t write_other; uint64_t write_self; - // Stats from compaction jobs - bytes written, bytes read, duration. - uint64_t compact_bytes_write; - uint64_t compact_bytes_read; - uint64_t compact_micros; // Total number of keys written. write_self and write_other measure number // of write requests written, Each of the write request can contain updates // to multiple keys. num_keys_written is total number of keys updated by all @@ -283,9 +288,6 @@ class InternalStats { write_with_wal(0), write_other(0), write_self(0), - compact_bytes_write(0), - compact_bytes_read(0), - compact_micros(0), num_keys_written(0), write_stall_micros(0), seconds_up(0) {}