diff --git a/db/compaction.cc b/db/compaction.cc index 768306ed4..5c382d163 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -415,14 +415,15 @@ void Compaction::Summary(char* output, int len) { uint64_t Compaction::OutputFilePreallocationSize() const { uint64_t preallocation_size = 0; - if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel || - output_level() > 0) { + if (max_output_file_size_ != port::kMaxUint64 && + (cfd_->ioptions()->compaction_style == kCompactionStyleLevel || + output_level() > 0)) { preallocation_size = max_output_file_size_; } else { - // output_level() == 0 - assert(num_input_levels() > 0); - for (const auto& f : inputs_[0].files) { - preallocation_size += f->fd.GetFileSize(); + for (const auto& level_files : inputs_) { + for (const auto& file : level_files.files) { + preallocation_size += file->fd.GetFileSize(); + } } } // Over-estimate slightly so we don't end up just barely crossing diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index cc0b01e01..e6d820ad4 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -780,6 +780,41 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) { ASSERT_OK(Put("", "")); } +TEST_F(DBCompactionTest, ManualCompactionUnknownOutputSize) { + // github issue #2249 + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + DestroyAndReopen(options); + + // create two files in l1 that we can compact + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < options.level0_file_num_compaction_trigger; j++) { + // make l0 files' ranges overlap to avoid trivial move + Put(std::to_string(2 * i), std::string(1, 'A')); + Put(std::to_string(2 * i + 1), std::string(1, 'A')); + Flush(); + dbfull()->TEST_WaitForFlushMemTable(); + } + dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 0), i + 1); + } + + ColumnFamilyMetaData cf_meta; + dbfull()->GetColumnFamilyMetaData(dbfull()->DefaultColumnFamily(), &cf_meta); + ASSERT_EQ(2, cf_meta.levels[1].files.size()); + std::vector input_filenames; + for (const auto& sst_file : cf_meta.levels[1].files) { + input_filenames.push_back(sst_file.name); + } + + // note CompactionOptions::output_file_size_limit is unset. + CompactionOptions compact_opt; + compact_opt.compression = kNoCompression; + dbfull()->CompactFiles(compact_opt, input_filenames, 1); +} + // Check that writes done during a memtable compaction are recovered // if the database is shutdown during the memtable compaction. TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) { diff --git a/db/db_test_util.h b/db/db_test_util.h index 65e983a50..03cb4f245 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -277,6 +277,9 @@ class SpecialEnv : public EnvWrapper { bool use_direct_io() const override { return base_->use_direct_io(); } + Status Allocate(uint64_t offset, uint64_t len) override { + return base_->Allocate(offset, len); + } }; class ManifestFile : public WritableFile { public: diff --git a/include/rocksdb/env.h b/include/rocksdb/env.h index a2fde4733..9af0261fa 100644 --- a/include/rocksdb/env.h +++ b/include/rocksdb/env.h @@ -703,14 +703,12 @@ class WritableFile { } } - protected: - /* - * Pre-allocate space for a file. - */ + // Pre-allocates space for a file. virtual Status Allocate(uint64_t offset, uint64_t len) { return Status::OK(); } + protected: size_t preallocation_block_size() { return preallocation_block_size_; } private: