From 3d9d77d9006c246ea54656440fb29eebfa048f8b Mon Sep 17 00:00:00 2001 From: Thomas Fersch Date: Wed, 22 May 2019 23:38:09 -0700 Subject: [PATCH] Restrict L0->L0 compaction according to max_compaction_bytes option (#5329) Summary: Modified FindIntraL0Compaction to stop picking more files if total amount of compensated bytes would be larger than max_compaction_bytes option. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5329 Differential Revision: D15435728 Pulled By: ThomasFersch fbshipit-source-id: d118a6da88d5df8ee20944422ade37cf6b15d60c --- db/compaction_picker.cc | 16 +++++++--- db/compaction_picker.h | 17 +++++++++++ db/compaction_picker_fifo.cc | 3 +- db/compaction_picker_test.cc | 59 ++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/db/compaction_picker.cc b/db/compaction_picker.cc index d6d7b6987..4bd8ff0e3 100644 --- a/db/compaction_picker.cc +++ b/db/compaction_picker.cc @@ -42,19 +42,23 @@ uint64_t TotalCompensatedFileSize(const std::vector& files) { bool FindIntraL0Compaction(const std::vector& level_files, size_t min_files_to_compact, uint64_t max_compact_bytes_per_del_file, + uint64_t max_compaction_bytes, CompactionInputFiles* comp_inputs) { size_t compact_bytes = static_cast(level_files[0]->fd.file_size); + uint64_t compensated_compact_bytes = level_files[0]->compensated_file_size; size_t compact_bytes_per_del_file = port::kMaxSizet; - // compaction range will be [0, span_len). + // Compaction range will be [0, span_len). size_t span_len; - // pull in files until the amount of compaction work per deleted file begins - // increasing. + // Pull in files until the amount of compaction work per deleted file begins + // increasing or maximum total compaction size is reached. size_t new_compact_bytes_per_del_file = 0; for (span_len = 1; span_len < level_files.size(); ++span_len) { compact_bytes += static_cast(level_files[span_len]->fd.file_size); + compensated_compact_bytes += level_files[span_len]->compensated_file_size; new_compact_bytes_per_del_file = compact_bytes / span_len; if (level_files[span_len]->being_compacted || - new_compact_bytes_per_del_file > compact_bytes_per_del_file) { + new_compact_bytes_per_del_file > compact_bytes_per_del_file || + compensated_compact_bytes > max_compaction_bytes) { break; } compact_bytes_per_del_file = new_compact_bytes_per_del_file; @@ -1627,7 +1631,9 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() { return false; } return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction, - port::kMaxUint64, &start_level_inputs_); + port::kMaxUint64, + mutable_cf_options_.max_compaction_bytes, + &start_level_inputs_); } } // namespace diff --git a/db/compaction_picker.h b/db/compaction_picker.h index 01f5495e6..250566b10 100644 --- a/db/compaction_picker.h +++ b/db/compaction_picker.h @@ -273,9 +273,26 @@ class NullCompactionPicker : public CompactionPicker { }; #endif // !ROCKSDB_LITE +// Attempts to find an intra L0 compaction conforming to the given parameters. +// +// @param level_files Metadata for L0 files. +// @param min_files_to_compact Minimum number of files required to +// do the compaction. +// @param max_compact_bytes_per_del_file Maximum average size in bytes per +// file that is going to get deleted by +// the compaction. +// @param max_compaction_bytes Maximum total size in bytes (in terms +// of compensated file size) for files +// to be compacted. +// @param [out] comp_inputs If a compaction was found, will be +// initialized with corresponding input +// files. Cannot be nullptr. +// +// @return true iff compaction was found. bool FindIntraL0Compaction(const std::vector& level_files, size_t min_files_to_compact, uint64_t max_compact_bytes_per_del_file, + uint64_t max_compaction_bytes, CompactionInputFiles* comp_inputs); CompressionType GetCompressionType(const ImmutableCFOptions& ioptions, diff --git a/db/compaction_picker_fifo.cc b/db/compaction_picker_fifo.cc index 1322989e5..eadb31f9e 100644 --- a/db/compaction_picker_fifo.cc +++ b/db/compaction_picker_fifo.cc @@ -134,7 +134,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction( mutable_cf_options .level0_file_num_compaction_trigger /* min_files_to_compact */ , - max_compact_bytes_per_del_file, &comp_inputs)) { + max_compact_bytes_per_del_file, + mutable_cf_options.max_compaction_bytes, &comp_inputs)) { Compaction* c = new Compaction( vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */, diff --git a/db/compaction_picker_test.cc b/db/compaction_picker_test.cc index 31325c128..c759dae8b 100644 --- a/db/compaction_picker_test.cc +++ b/db/compaction_picker_test.cc @@ -1478,6 +1478,65 @@ TEST_F(CompactionPickerTest, CacheNextCompactionIndex) { ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */)); } +TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesNotHit) { + // Intra L0 compaction triggers only if there are at least + // level0_file_num_compaction_trigger + 2 L0 files. + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_compaction_bytes = 1000000u; + NewVersionStorage(6, kCompactionStyleLevel); + + // All 5 L0 files will be picked for intra L0 compaction. The one L1 file + // spans entire L0 key range and is marked as being compacted to avoid + // L0->L1 compaction. + Add(0, 1U, "100", "150", 200000U); + Add(0, 2U, "151", "200", 200000U); + Add(0, 3U, "201", "250", 200000U); + Add(0, 4U, "251", "300", 200000U); + Add(0, 5U, "301", "350", 200000U); + Add(1, 6U, "100", "350", 200000U); + vstorage_->LevelFiles(1)[0]->being_compacted = true; + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_levels()); + ASSERT_EQ(5U, compaction->num_input_files(0)); + ASSERT_EQ(CompactionReason::kLevelL0FilesNum, + compaction->compaction_reason()); + ASSERT_EQ(0U, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesHit) { + // Intra L0 compaction triggers only if there are at least + // level0_file_num_compaction_trigger + 2 L0 files. + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_compaction_bytes = 999999u; + NewVersionStorage(6, kCompactionStyleLevel); + + // 4 out of 5 L0 files will be picked for intra L0 compaction due to + // max_compaction_bytes limit (the minimum number of files for triggering + // intra L0 compaction is 4). The one L1 file spans entire L0 key range and + // is marked as being compacted to avoid L0->L1 compaction. + Add(0, 1U, "100", "150", 200000U); + Add(0, 2U, "151", "200", 200000U); + Add(0, 3U, "201", "250", 200000U); + Add(0, 4U, "251", "300", 200000U); + Add(0, 5U, "301", "350", 200000U); + Add(1, 6U, "100", "350", 200000U); + vstorage_->LevelFiles(1)[0]->being_compacted = true; + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_levels()); + ASSERT_EQ(4U, compaction->num_input_files(0)); + ASSERT_EQ(CompactionReason::kLevelL0FilesNum, + compaction->compaction_reason()); + ASSERT_EQ(0U, compaction->output_level()); +} + } // namespace rocksdb int main(int argc, char** argv) {