diff --git a/db/compaction.cc b/db/compaction.cc index 6126b4642..c2785adee 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -405,20 +405,23 @@ void Compaction::Summary(char* output, int len) { uint64_t Compaction::OutputFilePreallocationSize() const { uint64_t preallocation_size = 0; + for (const auto& level_files : inputs_) { + for (const auto& file : level_files.files) { + preallocation_size += file->fd.GetFileSize(); + } + } + if (max_output_file_size_ != port::kMaxUint64 && - (cfd_->ioptions()->compaction_style == kCompactionStyleLevel || + (immutable_cf_options_.compaction_style == kCompactionStyleLevel || output_level() > 0)) { - preallocation_size = max_output_file_size_; - } else { - for (const auto& level_files : inputs_) { - for (const auto& file : level_files.files) { - preallocation_size += file->fd.GetFileSize(); - } - } + preallocation_size = std::min(max_output_file_size_, preallocation_size); } + // Over-estimate slightly so we don't end up just barely crossing // the threshold - return preallocation_size + (preallocation_size / 10); + // No point to prellocate more than 1GB. + return std::min(uint64_t{1073741824}, + preallocation_size + (preallocation_size / 10)); } std::unique_ptr Compaction::CreateCompactionFilter() const { diff --git a/db/compaction_picker_test.cc b/db/compaction_picker_test.cc index 069308bcc..4752d3428 100644 --- a/db/compaction_picker_test.cc +++ b/db/compaction_picker_test.cc @@ -175,6 +175,8 @@ TEST_F(CompactionPickerTest, Level1Trigger) { } TEST_F(CompactionPickerTest, Level1Trigger2) { + mutable_cf_options_.target_file_size_base = 10000000000; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); NewVersionStorage(6, kCompactionStyleLevel); Add(1, 66U, "150", "200", 1000000001U); Add(1, 88U, "201", "300", 1000000000U); @@ -191,13 +193,14 @@ TEST_F(CompactionPickerTest, Level1Trigger2) { ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize()); } TEST_F(CompactionPickerTest, LevelMaxScore) { NewVersionStorage(6, kCompactionStyleLevel); mutable_cf_options_.target_file_size_base = 10000000; - mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); Add(0, 1U, "150", "200", 1000000U); // Level 1 score 1.2 Add(1, 66U, "150", "200", 6000000U); @@ -218,6 +221,9 @@ TEST_F(CompactionPickerTest, LevelMaxScore) { ASSERT_TRUE(compaction.get() != nullptr); ASSERT_EQ(1U, compaction->num_input_files(0)); ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(mutable_cf_options_.target_file_size_base + + mutable_cf_options_.target_file_size_base / 10, + compaction->OutputFilePreallocationSize()); } TEST_F(CompactionPickerTest, NeedsCompactionLevel) { @@ -521,9 +527,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { NewVersionStorage(6, kCompactionStyleLevel); ioptions_.compaction_pri = kMinOverlappingRatio; - mutable_cf_options_.target_file_size_base = 10000000; + mutable_cf_options_.target_file_size_base = 100000000000; mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); Add(2, 6U, "150", "179", 50000000U); Add(2, 7U, "180", "220", 50000000U); @@ -543,6 +550,8 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { ASSERT_EQ(1U, compaction->num_input_files(0)); // Pick file 8 because it overlaps with 0 files on level 3. ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); + // Compaction input size * 1.1 + ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize()); } TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {