From 56ed601df30e6a0352462235eb7904738323cb89 Mon Sep 17 00:00:00 2001 From: Aaron Kabcenell Date: Wed, 29 Jul 2020 13:38:07 -0700 Subject: [PATCH] Compaction Read/Write Stats by Compaction Type (#7165) Summary: Adds compaction statistics (total bytes read and written) for compactions that occur for delete-triggered, periodic, and TTL compaction reasons. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7165 Test Plan: TTL and periodic can be checked by runnning db_bench with the options activated: /db_bench --benchmarks="fillrandom,stats" --statistics --num=10000000 -base_background_compactions=16 -periodic_compaction_seconds=1 ./db_bench --benchmarks="fillrandom,stats" --statistics --num=10000000 -base_background_compactions=16 -fifo_compaction_ttl=1 Setting the time to one second causes non-zero bytes read/written for those compaction reasons. Disabling them or setting them to times longer than the test run length causes the stats to return to zero as expected. Delete-triggered compaction counting is tested in DBTablePropertiesTest.DeletionTriggeredCompactionMarking Reviewed By: ajkr Differential Revision: D22693050 Pulled By: akabcenell fbshipit-source-id: d15cef4d94576f703015c8942d5f0d492f69401d --- db/compaction/compaction_job.cc | 14 ++++++++++- db/db_table_properties_test.cc | 3 +++ include/rocksdb/statistics.h | 8 +++++++ java/rocksjni/portal.h | 24 +++++++++++++++++++ .../src/main/java/org/rocksdb/TickerType.java | 10 ++++++++ monitoring/statistics.cc | 6 +++++ tools/db_bench_tool.cc | 6 +++++ 7 files changed, 70 insertions(+), 1 deletion(-) diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 49b94ba64..933524d1c 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -1509,10 +1509,22 @@ Status CompactionJob::InstallCompactionResults( void CompactionJob::RecordCompactionIOStats() { RecordTick(stats_, COMPACT_READ_BYTES, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES, IOSTATS(bytes_written)); + CompactionReason compaction_reason = + compact_->compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kFilesMarkedForCompaction) { + RecordTick(stats_, COMPACT_READ_BYTES_MARKED, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES_MARKED, IOSTATS(bytes_written)); + } else if (compaction_reason == CompactionReason::kPeriodicCompaction) { + RecordTick(stats_, COMPACT_READ_BYTES_PERIODIC, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES_PERIODIC, IOSTATS(bytes_written)); + } else if (compaction_reason == CompactionReason::kTtl) { + RecordTick(stats_, COMPACT_READ_BYTES_TTL, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES_TTL, IOSTATS(bytes_written)); + } ThreadStatusUtil::IncreaseThreadOperationProperty( ThreadStatus::COMPACTION_BYTES_READ, IOSTATS(bytes_read)); IOSTATS_RESET(bytes_read); - RecordTick(stats_, COMPACT_WRITE_BYTES, IOSTATS(bytes_written)); ThreadStatusUtil::IncreaseThreadOperationProperty( ThreadStatus::COMPACTION_BYTES_WRITTEN, IOSTATS(bytes_written)); IOSTATS_RESET(bytes_written); diff --git a/db/db_table_properties_test.cc b/db/db_table_properties_test.cc index cf71cc94a..add2200d2 100644 --- a/db/db_table_properties_test.cc +++ b/db/db_table_properties_test.cc @@ -294,6 +294,7 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) { NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger); Options opts = CurrentOptions(); + opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); opts.table_properties_collector_factories.emplace_back(compact_on_del); if(GetParam() == "kCompactionStyleUniversal") { @@ -364,6 +365,8 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) { dbfull()->TEST_WaitForCompact(); ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_LT(0, opts.statistics->getTickerCount(COMPACT_WRITE_BYTES_MARKED)); + ASSERT_LT(0, opts.statistics->getTickerCount(COMPACT_READ_BYTES_MARKED)); } INSTANTIATE_TEST_CASE_P( diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index af09a1b8d..cc99f6763 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -205,6 +205,14 @@ enum Tickers : uint32_t { COMPACT_WRITE_BYTES, // Bytes written during compaction FLUSH_WRITE_BYTES, // Bytes written during flush + // Compaction read and write statistics broken down by CompactionReason + COMPACT_READ_BYTES_MARKED, + COMPACT_READ_BYTES_PERIODIC, + COMPACT_READ_BYTES_TTL, + COMPACT_WRITE_BYTES_MARKED, + COMPACT_WRITE_BYTES_PERIODIC, + COMPACT_WRITE_BYTES_TTL, + // Number of table's properties loaded directly from file, without creating // table reader object. NUMBER_DIRECT_LOAD_TABLE_PROPERTIES, diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index 9bc1e674e..7a54e387f 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -4949,6 +4949,18 @@ class TickerTypeJni { return -0x0E; case ROCKSDB_NAMESPACE::Tickers::FILES_DELETED_IMMEDIATELY: return -0X0F; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_MARKED: + return -0x10; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_PERIODIC: + return -0x11; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_TTL: + return -0x12; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_MARKED: + return -0x13; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_PERIODIC: + return -0x14; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_TTL: + return -0x15; case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX: // 0x5F for backwards compatibility on current minor version. @@ -5249,6 +5261,18 @@ class TickerTypeJni { return ROCKSDB_NAMESPACE::Tickers::FILES_MARKED_TRASH; case -0x0F: return ROCKSDB_NAMESPACE::Tickers::FILES_DELETED_IMMEDIATELY; + case -0x10: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_MARKED; + case -0x11: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_PERIODIC; + case -0x12: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_TTL; + case -0x13: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_MARKED; + case -0x14: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_PERIODIC; + case -0x15: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_TTL; case 0x5F: // 0x5F for backwards compatibility on current minor version. return ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX; diff --git a/java/src/main/java/org/rocksdb/TickerType.java b/java/src/main/java/org/rocksdb/TickerType.java index 095178df6..7a37f35b9 100644 --- a/java/src/main/java/org/rocksdb/TickerType.java +++ b/java/src/main/java/org/rocksdb/TickerType.java @@ -732,6 +732,16 @@ public enum TickerType { */ FILES_DELETED_IMMEDIATELY((byte) -0x0f), + /** + * Compaction read and write statistics broken down by CompactionReason + */ + COMPACT_READ_BYTES_MARKED((byte) -0x10), + COMPACT_READ_BYTES_PERIODIC((byte) -0x11), + COMPACT_READ_BYTES_TTL((byte) -0x12), + COMPACT_WRITE_BYTES_MARKED((byte) -0x13), + COMPACT_WRITE_BYTES_PERIODIC((byte) -0x14), + COMPACT_WRITE_BYTES_TTL((byte) -0x15), + TICKER_ENUM_MAX((byte) 0x5F); private final byte value; diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index cc523d58b..181065c43 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -105,6 +105,12 @@ const std::vector> TickersNameMap = { {COMPACT_READ_BYTES, "rocksdb.compact.read.bytes"}, {COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes"}, {FLUSH_WRITE_BYTES, "rocksdb.flush.write.bytes"}, + {COMPACT_READ_BYTES_MARKED, "rocksdb.compact.read.marked.bytes"}, + {COMPACT_READ_BYTES_PERIODIC, "rocksdb.compact.read.periodic.bytes"}, + {COMPACT_READ_BYTES_TTL, "rocksdb.compact.read.ttl.bytes"}, + {COMPACT_WRITE_BYTES_MARKED, "rocksdb.compact.write.marked.bytes"}, + {COMPACT_WRITE_BYTES_PERIODIC, "rocksdb.compact.write.periodic.bytes"}, + {COMPACT_WRITE_BYTES_TTL, "rocksdb.compact.write.ttl.bytes"}, {NUMBER_DIRECT_LOAD_TABLE_PROPERTIES, "rocksdb.number.direct.load.table.properties"}, {NUMBER_SUPERVERSION_ACQUIRES, "rocksdb.number.superversion_acquires"}, diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 33386fb79..16016c6ee 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -685,6 +685,11 @@ DEFINE_int32(level0_file_num_compaction_trigger, "Number of files in level-0" " when compactions start"); +DEFINE_uint64(periodic_compaction_seconds, + ROCKSDB_NAMESPACE::Options().periodic_compaction_seconds, + "Files older than this will be picked up for compaction and" + " rewritten to the same level"); + static bool ValidateInt32Percent(const char* flagname, int32_t value) { if (value <= 0 || value>=100) { fprintf(stderr, "Invalid value for --%s: %d, 0< pct <100 \n", @@ -3962,6 +3967,7 @@ class Benchmark { options.max_compaction_bytes = FLAGS_max_compaction_bytes; options.disable_auto_compactions = FLAGS_disable_auto_compactions; options.optimize_filters_for_hits = FLAGS_optimize_filters_for_hits; + options.periodic_compaction_seconds = FLAGS_periodic_compaction_seconds; // fill storage options options.advise_random_on_open = FLAGS_advise_random_on_open;