diff --git a/db/compaction_iteration_stats.h b/db/compaction_iteration_stats.h new file mode 100644 index 000000000..f11369ec1 --- /dev/null +++ b/db/compaction_iteration_stats.h @@ -0,0 +1,29 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#pragma once + +struct CompactionIterationStats { + // Compaction statistics + int64_t num_record_drop_user = 0; + int64_t num_record_drop_hidden = 0; + int64_t num_record_drop_obsolete = 0; + int64_t num_record_drop_range_del = 0; + int64_t num_range_del_drop_obsolete = 0; + uint64_t total_filter_time = 0; + + // Input statistics + // TODO(noetzli): The stats are incomplete. They are lacking everything + // consumed by MergeHelper. + uint64_t num_input_records = 0; + uint64_t num_input_deletion_records = 0; + uint64_t num_input_corrupt_records = 0; + uint64_t total_input_raw_key_bytes = 0; + uint64_t total_input_raw_value_bytes = 0; + + // Single-Delete diagnostics for exceptional situations + uint64_t num_single_del_fallthru = 0; + uint64_t num_single_del_mismatch = 0; +}; diff --git a/db/compaction_iterator.cc b/db/compaction_iterator.cc index 58503dd15..14be6e493 100644 --- a/db/compaction_iterator.cc +++ b/db/compaction_iterator.cc @@ -63,6 +63,8 @@ void CompactionIterator::ResetRecordCounts() { iter_stats_.num_record_drop_user = 0; iter_stats_.num_record_drop_hidden = 0; iter_stats_.num_record_drop_obsolete = 0; + iter_stats_.num_record_drop_range_del = 0; + iter_stats_.num_range_del_drop_obsolete = 0; } void CompactionIterator::SeekToFirst() { @@ -424,6 +426,8 @@ void CompactionIterator::NextFromInput() { // 2. different snapshot stripe bool should_delete = range_del_agg_->ShouldDelete(key_); if (should_delete) { + ++iter_stats_.num_record_drop_hidden; + ++iter_stats_.num_record_drop_range_del; input_->Next(); } else { valid_ = true; diff --git a/db/compaction_iterator.h b/db/compaction_iterator.h index 6ede0d66d..d22ca01b5 100644 --- a/db/compaction_iterator.h +++ b/db/compaction_iterator.h @@ -12,6 +12,7 @@ #include #include "db/compaction.h" +#include "db/compaction_iteration_stats.h" #include "db/merge_helper.h" #include "db/pinned_iterators_manager.h" #include "db/range_del_aggregator.h" @@ -20,27 +21,6 @@ namespace rocksdb { -struct CompactionIteratorStats { - // Compaction statistics - int64_t num_record_drop_user = 0; - int64_t num_record_drop_hidden = 0; - int64_t num_record_drop_obsolete = 0; - uint64_t total_filter_time = 0; - - // Input statistics - // TODO(noetzli): The stats are incomplete. They are lacking everything - // consumed by MergeHelper. - uint64_t num_input_records = 0; - uint64_t num_input_deletion_records = 0; - uint64_t num_input_corrupt_records = 0; - uint64_t total_input_raw_key_bytes = 0; - uint64_t total_input_raw_value_bytes = 0; - - // Single-Delete diagnostics for exceptional situations - uint64_t num_single_del_fallthru = 0; - uint64_t num_single_del_mismatch = 0; -}; - class CompactionIterator { public: CompactionIterator(InternalIterator* input, const Comparator* cmp, @@ -74,7 +54,7 @@ class CompactionIterator { const ParsedInternalKey& ikey() const { return ikey_; } bool Valid() const { return valid_; } const Slice& user_key() const { return current_user_key_; } - const CompactionIteratorStats& iter_stats() const { return iter_stats_; } + const CompactionIterationStats& iter_stats() const { return iter_stats_; } private: // Processes the input stream to find the next output @@ -157,6 +137,6 @@ class CompactionIterator { // increasing so a later call to the function must be looking for a key that // is in or beyond the last file checked during the previous call std::vector level_ptrs_; - CompactionIteratorStats iter_stats_; + CompactionIterationStats iter_stats_; }; } // namespace rocksdb diff --git a/db/compaction_job.cc b/db/compaction_job.cc index c4fb012b5..844a8fffd 100644 --- a/db/compaction_job.cc +++ b/db/compaction_job.cc @@ -768,8 +768,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { sub_compact->ShouldStopBefore( key, sub_compact->current_output_file_size) && sub_compact->builder != nullptr) { - status = FinishCompactionOutputFile(input->status(), sub_compact, - range_del_agg.get()); + CompactionIterationStats range_del_out_stats; + status = + FinishCompactionOutputFile(input->status(), sub_compact, + range_del_agg.get(), &range_del_out_stats); + RecordDroppedKeys(range_del_out_stats, + &sub_compact->compaction_job_stats); if (!status.ok()) { break; } @@ -861,8 +865,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { if (c_iter->Valid()) { next_key = &c_iter->key(); } + CompactionIterationStats range_del_out_stats; status = FinishCompactionOutputFile(input_status, sub_compact, - range_del_agg.get(), next_key); + range_del_agg.get(), + &range_del_out_stats, next_key); + RecordDroppedKeys(range_del_out_stats, + &sub_compact->compaction_job_stats); if (sub_compact->outputs.size() == 1) { // Use dictionary from first output file for compression of subsequent // files. @@ -902,8 +910,11 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { status = OpenCompactionOutputFile(sub_compact); } if (status.ok() && sub_compact->builder != nullptr) { - status = FinishCompactionOutputFile(input->status(), sub_compact, - range_del_agg.get()); + CompactionIterationStats range_del_out_stats; + status = + FinishCompactionOutputFile(input->status(), sub_compact, + range_del_agg.get(), &range_del_out_stats); + RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats); } if (status.ok()) { status = input->status(); @@ -929,7 +940,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { } void CompactionJob::RecordDroppedKeys( - const CompactionIteratorStats& c_iter_stats, + const CompactionIterationStats& c_iter_stats, CompactionJobStats* compaction_job_stats) { if (c_iter_stats.num_record_drop_user > 0) { RecordTick(stats_, COMPACTION_KEY_DROP_USER, @@ -951,11 +962,20 @@ void CompactionJob::RecordDroppedKeys( c_iter_stats.num_record_drop_obsolete; } } + if (c_iter_stats.num_record_drop_range_del > 0) { + RecordTick(stats_, COMPACTION_KEY_DROP_RANGE_DEL, + c_iter_stats.num_record_drop_range_del); + } + if (c_iter_stats.num_range_del_drop_obsolete > 0) { + RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE, + c_iter_stats.num_range_del_drop_obsolete); + } } Status CompactionJob::FinishCompactionOutputFile( const Status& input_status, SubcompactionState* sub_compact, RangeDelAggregator* range_del_agg, + CompactionIterationStats* range_del_out_stats, const Slice* next_table_min_key /* = nullptr */) { AutoThreadOperationStageUpdater stage_updater( ThreadStatus::STAGE_COMPACTION_SYNC_FILE); @@ -998,7 +1018,8 @@ Status CompactionJob::FinishCompactionOutputFile( upper_bound = sub_compact->end; } range_del_agg->AddToBuilder(sub_compact->builder.get(), lower_bound, - upper_bound, meta, bottommost_level_); + upper_bound, meta, range_del_out_stats, + bottommost_level_); } const uint64_t current_entries = sub_compact->builder->NumEntries(); meta->marked_for_compaction = sub_compact->builder->NeedCompact(); diff --git a/db/compaction_job.h b/db/compaction_job.h index 12a57d71b..25199a6ff 100644 --- a/db/compaction_job.h +++ b/db/compaction_job.h @@ -96,17 +96,18 @@ class CompactionJob { // kv-pairs void ProcessKeyValueCompaction(SubcompactionState* sub_compact); - Status FinishCompactionOutputFile(const Status& input_status, - SubcompactionState* sub_compact, - RangeDelAggregator* range_del_agg = nullptr, - const Slice* next_table_min_key = nullptr); + Status FinishCompactionOutputFile( + const Status& input_status, SubcompactionState* sub_compact, + RangeDelAggregator* range_del_agg, + CompactionIterationStats* range_del_out_stats, + const Slice* next_table_min_key = nullptr); Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options); void RecordCompactionIOStats(); Status OpenCompactionOutputFile(SubcompactionState* sub_compact); void CleanupCompaction(); void UpdateCompactionJobStats( const InternalStats::CompactionStats& stats) const; - void RecordDroppedKeys(const CompactionIteratorStats& c_iter_stats, + void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats, CompactionJobStats* compaction_job_stats = nullptr); void UpdateCompactionStats(); diff --git a/db/range_del_aggregator.cc b/db/range_del_aggregator.cc index 3ea8cda4f..b97c426b8 100644 --- a/db/range_del_aggregator.cc +++ b/db/range_del_aggregator.cc @@ -140,6 +140,7 @@ RangeDelAggregator::TombstoneMap& RangeDelAggregator::GetTombstoneMap( void RangeDelAggregator::AddToBuilder( TableBuilder* builder, const Slice* lower_bound, const Slice* upper_bound, FileMetaData* meta, + CompactionIterationStats* range_del_out_stats /* = nullptr */, bool bottommost_level /* = false */) { if (rep_ == nullptr) { return; @@ -147,6 +148,10 @@ void RangeDelAggregator::AddToBuilder( auto stripe_map_iter = rep_->stripe_map_.begin(); assert(stripe_map_iter != rep_->stripe_map_.end()); if (bottommost_level) { + range_del_out_stats->num_range_del_drop_obsolete += + static_cast(stripe_map_iter->second.size()); + range_del_out_stats->num_record_drop_obsolete += + static_cast(stripe_map_iter->second.size()); // For the bottommost level, keys covered by tombstones in the first // (oldest) stripe have been compacted away, so the tombstones are obsolete. ++stripe_map_iter; diff --git a/db/range_del_aggregator.h b/db/range_del_aggregator.h index 403a6033f..47c29a043 100644 --- a/db/range_del_aggregator.h +++ b/db/range_del_aggregator.h @@ -9,6 +9,7 @@ #include #include +#include "db/compaction_iteration_stats.h" #include "db/dbformat.h" #include "db/pinned_iterators_manager.h" #include "db/version_edit.h" @@ -81,6 +82,7 @@ class RangeDelAggregator { // compaction. void AddToBuilder(TableBuilder* builder, const Slice* lower_bound, const Slice* upper_bound, FileMetaData* meta, + CompactionIterationStats* range_del_out_stats = nullptr, bool bottommost_level = false); bool IsEmpty(); diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index 25bc4a8ec..0271dd1a4 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -95,12 +95,16 @@ enum Tickers : uint32_t { /** * COMPACTION_KEY_DROP_* count the reasons for key drop during compaction - * There are 3 reasons currently. + * There are 4 reasons currently. */ COMPACTION_KEY_DROP_NEWER_ENTRY, // key was written with a newer value. + // Also includes keys dropped for range del. COMPACTION_KEY_DROP_OBSOLETE, // The key is obsolete. + COMPACTION_KEY_DROP_RANGE_DEL, // key was covered by a range tombstone. COMPACTION_KEY_DROP_USER, // user compaction function has dropped the key. + COMPACTION_RANGE_DEL_DROP_OBSOLETE, // all keys in range were deleted. + // Number of keys written to the database via the Put and Write call's NUMBER_KEYS_WRITTEN, // Number of Keys read, @@ -251,7 +255,10 @@ const std::vector> TickersNameMap = { {GET_HIT_L2_AND_UP, "rocksdb.l2andup.hit"}, {COMPACTION_KEY_DROP_NEWER_ENTRY, "rocksdb.compaction.key.drop.new"}, {COMPACTION_KEY_DROP_OBSOLETE, "rocksdb.compaction.key.drop.obsolete"}, + {COMPACTION_KEY_DROP_RANGE_DEL, "rocksdb.compaction.key.drop.range_del"}, {COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user"}, + {COMPACTION_RANGE_DEL_DROP_OBSOLETE, + "rocksdb.compaction.range_del.drop.obsolete"}, {NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written"}, {NUMBER_KEYS_READ, "rocksdb.number.keys.read"}, {NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated"},