diff --git a/db/compaction.cc b/db/compaction.cc index 786d1f578..20a8ba40d 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -36,83 +36,91 @@ void Compaction::SetInputVersion(Version* _input_version) { cfd_->Ref(); input_version_->Ref(); - edit_ = new VersionEdit(); - edit_->SetColumnFamily(cfd_->GetID()); + edit_.SetColumnFamily(cfd_->GetID()); } -Compaction::Compaction(int number_levels, int _start_level, int out_level, - uint64_t target_file_size, - uint64_t max_grandparent_overlap_bytes, - uint32_t output_path_id, - CompressionType output_compression, bool seek_compaction, - bool deletion_compaction) - : start_level_(_start_level), - output_level_(out_level), - max_output_file_size_(target_file_size), - max_grandparent_overlap_bytes_(max_grandparent_overlap_bytes), - input_version_(nullptr), - edit_(nullptr), - number_levels_(number_levels), - cfd_(nullptr), - output_path_id_(output_path_id), - output_compression_(output_compression), - seek_compaction_(seek_compaction), - deletion_compaction_(deletion_compaction), - grandparent_index_(0), - seen_key_(false), - overlapped_bytes_(0), - base_index_(-1), - parent_index_(-1), - score_(0), - bottommost_level_(false), - is_full_compaction_(false), - is_manual_compaction_(false), - level_ptrs_(std::vector(number_levels_)) { - for (int i = 0; i < number_levels_; i++) { - level_ptrs_[i] = 0; +// helper function to determine if compaction is creating files at the +// bottommost level +bool Compaction::IsBottommostLevel( + int output_level, VersionStorageInfo* vstorage, + const std::vector& inputs) { + if (inputs[0].level == 0 && + inputs[0].files.back() != vstorage->LevelFiles(0).back()) { + return false; + } + + // checks whether there are files living beyond the output_level. + for (int i = output_level + 1; i < vstorage->num_levels(); i++) { + if (vstorage->NumLevelFiles(i) > 0) { + return false; + } } - int num_levels = output_level_ - start_level_ + 1; - input_levels_.resize(num_levels); - inputs_.resize(num_levels); - for (int i = 0; i < num_levels; ++i) { - inputs_[i].level = start_level_ + i; + return true; +} + +bool Compaction::IsFullCompaction( + VersionStorageInfo* vstorage, + const std::vector& inputs) { + int num_files_in_compaction = 0; + int total_num_files = 0; + for (int l = 0; l < vstorage->num_levels(); l++) { + total_num_files += vstorage->NumLevelFiles(l); } + for (size_t i = 0; i < inputs.size(); i++) { + num_files_in_compaction += inputs[i].size(); + } + return num_files_in_compaction == total_num_files; } Compaction::Compaction(VersionStorageInfo* vstorage, - const autovector& _inputs, - int _start_level, int _output_level, - uint64_t _max_grandparent_overlap_bytes, - const CompactionOptions& _options, - bool _deletion_compaction) - : start_level_(_start_level), + const MutableCFOptions& _mutable_cf_options, + std::vector _inputs, + int _output_level, uint64_t _target_file_size, + uint64_t _max_grandparent_overlap_bytes, + uint32_t _output_path_id, CompressionType _compression, + std::vector _grandparents, + bool _manual_compaction, double _score, + bool _deletion_compaction) + : start_level_(_inputs[0].level), output_level_(_output_level), - max_output_file_size_(_options.output_file_size_limit), + max_output_file_size_(_target_file_size), max_grandparent_overlap_bytes_(_max_grandparent_overlap_bytes), + mutable_cf_options_(_mutable_cf_options), input_version_(nullptr), number_levels_(vstorage->num_levels()), cfd_(nullptr), - output_compression_(_options.compression), - seek_compaction_(false), + output_path_id_(_output_path_id), + output_compression_(_compression), deletion_compaction_(_deletion_compaction), - inputs_(_inputs), + inputs_(std::move(_inputs)), + grandparents_(std::move(_grandparents)), grandparent_index_(0), seen_key_(false), overlapped_bytes_(0), - base_index_(-1), - parent_index_(-1), - score_(0), - bottommost_level_(false), - is_full_compaction_(false), - is_manual_compaction_(false), - level_ptrs_(std::vector(number_levels_)) { - for (int i = 0; i < number_levels_; i++) { - level_ptrs_[i] = 0; + score_(_score), + bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)), + is_full_compaction_(IsFullCompaction(vstorage, inputs_)), + is_manual_compaction_(_manual_compaction), + level_ptrs_(std::vector(number_levels_, 0)) { + MarkFilesBeingCompacted(true); + +#ifndef NDEBUG + for (size_t i = 1; i < inputs_.size(); ++i) { + assert(inputs_[i].level > inputs_[i - 1].level); + } +#endif + + // setup input_levels_ + { + input_levels_.resize(num_input_levels()); + for (size_t which = 0; which < num_input_levels(); which++) { + DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files, + &arena_); + } } } Compaction::~Compaction() { - delete edit_; if (input_version_ != nullptr) { input_version_->Unref(); } @@ -123,14 +131,6 @@ Compaction::~Compaction() { } } -void Compaction::GenerateFileLevels() { - input_levels_.resize(num_input_levels()); - for (size_t which = 0; which < num_input_levels(); which++) { - DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files, - &arena_); - } -} - bool Compaction::InputCompressionMatchesOutput() const { int base_level = input_version_->storage_info()->base_level(); bool matches = (GetCompressionType(*cfd_->ioptions(), start_level_, @@ -144,25 +144,14 @@ bool Compaction::InputCompressionMatchesOutput() const { } bool Compaction::IsTrivialMove() const { - // If start_level_== output_level_, the purpose is to force compaction - // filter to be applied to that level, and thus cannot be a trivia move. - if (start_level_ == output_level_) { - return false; - } - // If compaction involves more than one file, it is not trivial move. - if (num_input_files(0) != 1) { - return false; - } - for (size_t l = 1u; l < num_input_levels(); l++) { - if (num_input_files(l) != 0) { - return false; - } - } - // Avoid a move if there is lots of overlapping grandparent data. // Otherwise, the move could create a parent file that will require // a very expensive merge later on. - return (input(0, 0)->fd.GetPathId() == GetOutputPathId() && + // If start_level_== output_level_, the purpose is to force compaction + // filter to be applied to that level, and thus cannot be a trivia move. + return (start_level_ != output_level_ && num_input_levels() == 1 && + num_input_files(0) == 1 && + input(0, 0)->fd.GetPathId() == GetOutputPathId() && InputCompressionMatchesOutput() && TotalFileSize(grandparents_) <= max_grandparent_overlap_bytes_); } @@ -240,31 +229,6 @@ void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) { } } -// Is this compaction producing files at the bottommost level? -void Compaction::SetupBottomMostLevel(VersionStorageInfo* vstorage, - bool is_manual, bool level0_only) { - if (level0_only) { - // If universal compaction style is used and manual - // compaction is occuring, then we are guaranteed that - // all files will be picked in a single compaction - // run. We can safely set bottommost_level_ = true. - // If it is not manual compaction, then bottommost_level_ - // is already set when the Compaction was created. - if (is_manual) { - bottommost_level_ = true; - } - return; - } - bottommost_level_ = true; - // checks whether there are files living beyond the output_level. - for (int i = output_level_ + 1; i < number_levels_; i++) { - if (vstorage->NumLevelFiles(i) > 0) { - bottommost_level_ = false; - break; - } - } -} - // Sample output: // If compacting 3 L0 files, 2 L3 files and 1 L4 file, and outputting to L5, // print: "3@0 + 2@3 + 1@4 files to L5" @@ -292,6 +256,7 @@ const char* Compaction::InputLevelSummary( } void Compaction::ReleaseCompactionFiles(Status status) { + MarkFilesBeingCompacted(false); cfd_->compaction_picker()->ReleaseCompactionFiles(this, status); } @@ -323,9 +288,9 @@ int InputSummary(const std::vector& files, char* output, void Compaction::Summary(char* output, int len) { int write = snprintf(output, len, "Base version %" PRIu64 - " Base level %d, seek compaction:%d, inputs: [", + " Base level %d, inputs: [", input_version_->GetVersionNumber(), - start_level_, seek_compaction_); + start_level_); if (write < 0 || write >= len) { return; } @@ -365,15 +330,4 @@ uint64_t Compaction::OutputFilePreallocationSize( return preallocation_size * 1.1; } -Compaction* Compaction::TEST_NewCompaction( - int num_levels, int start_level, int out_level, uint64_t target_file_size, - uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id, - CompressionType output_compression, bool seek_compaction, - bool deletion_compaction) { - return new Compaction(num_levels, start_level, out_level, target_file_size, - max_grandparent_overlap_bytes, output_path_id, - output_compression, seek_compaction, - deletion_compaction); -} - } // namespace rocksdb diff --git a/db/compaction.h b/db/compaction.h index bb2e70d06..876f91c30 100644 --- a/db/compaction.h +++ b/db/compaction.h @@ -34,11 +34,13 @@ class VersionStorageInfo; class Compaction { public: Compaction(VersionStorageInfo* input_version, - const autovector& inputs, - int start_level, int output_level, - uint64_t max_grandparent_overlap_bytes, - const CompactionOptions& options, - bool deletion_compaction); + const MutableCFOptions& mutable_cf_options, + std::vector inputs, int output_level, + uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes, + uint32_t output_path_id, CompressionType compression, + std::vector grandparents, + bool manual_compaction = false, double score = -1, + bool deletion_compaction = false); // No copying allowed Compaction(const Compaction&) = delete; @@ -62,7 +64,7 @@ class Compaction { // Return the object that holds the edits to the descriptor done // by this compaction. - VersionEdit* edit() const { return edit_; } + VersionEdit* edit() { return &edit_; } // Returns the number of input files associated to the specified // compaction input level. @@ -113,10 +115,6 @@ class Compaction { // Whether need to write output file to second DB path. uint32_t GetOutputPathId() const { return output_path_id_; } - // Generate input_levels_ from inputs_ - // Should be called when inputs_ is stable - void GenerateFileLevels(); - // Is this a trivial compaction that can be implemented by just // moving a single input file to the next level (no merging or splitting) bool IsTrivialMove() const; @@ -158,8 +156,6 @@ class Compaction { // Was this compaction triggered manually by the client? bool IsManualCompaction() { return is_manual_compaction_; } - void SetOutputPathId(uint32_t path_id) { output_path_id_ = path_id; } - // Return the MutableCFOptions that should be used throughout the compaction // procedure const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; } @@ -171,42 +167,27 @@ class Compaction { void SetInputVersion(Version* input_version); - // mark (or clear) all files that are being compacted - void MarkFilesBeingCompacted(bool mark_as_compacted); - - // Initialize whether the compaction is producing files at the - // bottommost level. - // - // @see BottomMostLevel() - void SetupBottomMostLevel(VersionStorageInfo* vstorage, bool is_manual, - bool level0_only); - - static Compaction* TEST_NewCompaction( - int num_levels, int start_level, int out_level, uint64_t target_file_size, - uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id, - CompressionType output_compression, bool seek_compaction = false, - bool deletion_compaction = false); - - CompactionInputFiles* TEST_GetInputFiles(int l) { - return &inputs_[l]; - } - struct InputLevelSummaryBuffer { char buffer[128]; }; const char* InputLevelSummary(InputLevelSummaryBuffer* scratch) const; + // In case of compaction error, reset the nextIndex that is used + // to pick up the next file to be compacted from files_by_size_ + void ResetNextCompactionIndex(); + private: - friend class CompactionPicker; - friend class UniversalCompactionPicker; - friend class FIFOCompactionPicker; - friend class LevelCompactionPicker; + // mark (or clear) all files that are being compacted + void MarkFilesBeingCompacted(bool mark_as_compacted); - Compaction(int num_levels, int start_level, int out_level, - uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes, - uint32_t output_path_id, CompressionType output_compression, - bool seek_compaction = false, bool deletion_compaction = false); + // helper function to determine if compaction with inputs and storage is + // bottommost + static bool IsBottommostLevel( + int output_level, VersionStorageInfo* vstorage, + const std::vector& inputs); + static bool IsFullCompaction(VersionStorageInfo* vstorage, + const std::vector& inputs); const int start_level_; // the lowest level to be compacted const int output_level_; // levels to which output files are stored @@ -214,43 +195,38 @@ class Compaction { uint64_t max_grandparent_overlap_bytes_; MutableCFOptions mutable_cf_options_; Version* input_version_; - VersionEdit* edit_; - int number_levels_; + VersionEdit edit_; + const int number_levels_; ColumnFamilyData* cfd_; Arena arena_; // Arena used to allocate space for file_levels_ - uint32_t output_path_id_; + const uint32_t output_path_id_; CompressionType output_compression_; - bool seek_compaction_; // If true, then the comaction can be done by simply deleting input files. - bool deletion_compaction_; + const bool deletion_compaction_; - // Compaction input files organized by level. - autovector inputs_; + // Compaction input files organized by level. Constant after construction + const std::vector inputs_; // A copy of inputs_, organized more closely in memory autovector input_levels_; // State used to check for number of of overlapping grandparent files // (grandparent == "output_level_ + 1") - // This vector is updated by Version::GetOverlappingInputs(). std::vector grandparents_; size_t grandparent_index_; // Index in grandparent_starts_ bool seen_key_; // Some output key has been seen uint64_t overlapped_bytes_; // Bytes of overlap between current output // and grandparent files - int base_index_; // index of the file in files_[start_level_] - int parent_index_; // index of some file with same range in - // files_[start_level_+1] - double score_; // score that was used to pick this compaction. + const double score_; // score that was used to pick this compaction. // Is this compaction creating a file in the bottom most level? - bool bottommost_level_; + const bool bottommost_level_; // Does this compaction include all sst files? - bool is_full_compaction_; + const bool is_full_compaction_; // Is this compaction requested by the client? - bool is_manual_compaction_; + const bool is_manual_compaction_; // "level_ptrs_" holds indices into "input_version_->levels_", where each // index remembers which file of an associated level we are currently used @@ -259,10 +235,6 @@ class Compaction { // records indices for all levels beyond "output_level_". std::vector level_ptrs_; - // In case of compaction error, reset the nextIndex that is used - // to pick up the next file to be compacted from files_by_size_ - void ResetNextCompactionIndex(); - // Does input compression match the output compression? bool InputCompressionMatchesOutput() const; }; diff --git a/db/compaction_job.cc b/db/compaction_job.cc index 07e3034aa..5c77d2321 100644 --- a/db/compaction_job.cc +++ b/db/compaction_job.cc @@ -245,7 +245,6 @@ void CompactionJob::Prepare() { auto* compaction = compact_->compaction; // Generate file_levels_ for compaction berfore making Iterator - compaction->GenerateFileLevels(); ColumnFamilyData* cfd = compact_->compaction->column_family_data(); assert(cfd != nullptr); { diff --git a/db/compaction_job_test.cc b/db/compaction_job_test.cc index 9b05bdff4..c5cbb5ad1 100644 --- a/db/compaction_job_test.cc +++ b/db/compaction_job_test.cc @@ -66,6 +66,9 @@ class CompactionJobTest : public testing::Test { auto key = ToString(i * (kKeysPerFile / 2) + k); auto value = ToString(i * kKeysPerFile + k); InternalKey internal_key(key, ++sequence_number, kTypeValue); + // This is how the key will look like once it's written in bottommost + // file + InternalKey bottommost_internal_key(key, 0, kTypeValue); if (k == 0) { smallest = internal_key; smallest_seqno = sequence_number; @@ -74,7 +77,7 @@ class CompactionJobTest : public testing::Test { largest_seqno = sequence_number; } std::pair key_value( - {internal_key.Encode().ToString(), value}); + {bottommost_internal_key.Encode().ToString(), value}); contents.insert(key_value); if (i == 1 || k < kKeysPerFile / 2) { expected_results.insert(key_value); @@ -143,15 +146,15 @@ TEST_F(CompactionJobTest, Simple) { auto files = cfd->current()->storage_info()->LevelFiles(0); ASSERT_EQ(2U, files.size()); - std::unique_ptr compaction(Compaction::TEST_NewCompaction( - 7, 0, 1, 1024 * 1024, 10, 0, kNoCompression)); + CompactionInputFiles compaction_input_files; + compaction_input_files.level = 0; + compaction_input_files.files.push_back(files[0]); + compaction_input_files.files.push_back(files[1]); + std::unique_ptr compaction(new Compaction( + cfd->current()->storage_info(), *cfd->GetLatestMutableCFOptions(), + {compaction_input_files}, 1, 1024 * 1024, 10, 0, kNoCompression, {})); compaction->SetInputVersion(cfd->current()); - auto compaction_input_files = compaction->TEST_GetInputFiles(0); - compaction_input_files->level = 0; - compaction_input_files->files.push_back(files[0]); - compaction_input_files->files.push_back(files[1]); - SnapshotList snapshots; int yield_callback_called = 0; std::function yield_callback = [&]() { diff --git a/db/compaction_picker.cc b/db/compaction_picker.cc index 8650b5cfd..540d3b405 100644 --- a/db/compaction_picker.cc +++ b/db/compaction_picker.cc @@ -70,11 +70,9 @@ CompactionPicker::CompactionPicker(const ImmutableCFOptions& ioptions, CompactionPicker::~CompactionPicker() {} -// Clear all files to indicate that they are not being compacted // Delete this compaction from the list of running compactions. void CompactionPicker::ReleaseCompactionFiles(Compaction* c, Status status) { - c->MarkFilesBeingCompacted(false); - if (c->level() == 0) { + if (c->start_level() == 0) { level0_compactions_in_progress_.erase(c); } if (!status.ok()) { @@ -113,61 +111,43 @@ void CompactionPicker::GetRange(const std::vector& inputs1, bool CompactionPicker::ExpandWhileOverlapping(const std::string& cf_name, VersionStorageInfo* vstorage, - Compaction* c) { - assert(c != nullptr); - // If inputs are empty then there is nothing to expand. - if (c->inputs_[0].empty()) { - assert(c->inputs(c->num_input_levels() - 1)->empty()); - // This isn't good compaction - return false; - } + CompactionInputFiles* inputs) { + // This isn't good compaction + assert(!inputs->empty()); + const int level = inputs->level; // GetOverlappingInputs will always do the right thing for level-0. // So we don't need to do any expansion if level == 0. - if (c->level() == 0) { + if (level == 0) { return true; } - const int level = c->level(); InternalKey smallest, largest; - // Keep expanding c->inputs_[0] until we are sure that there is a - // "clean cut" boundary between the files in input and the surrounding files. + // Keep expanding inputs until we are sure that there is a "clean cut" + // boundary between the files in input and the surrounding files. // This will ensure that no parts of a key are lost during compaction. int hint_index = -1; size_t old_size; do { - old_size = c->inputs_[0].size(); - GetRange(c->inputs_[0].files, &smallest, &largest); - c->inputs_[0].clear(); - vstorage->GetOverlappingInputs(level, &smallest, &largest, - &c->inputs_[0].files, hint_index, - &hint_index); - } while(c->inputs_[0].size() > old_size); + old_size = inputs->size(); + GetRange(inputs->files, &smallest, &largest); + inputs->clear(); + vstorage->GetOverlappingInputs(level, &smallest, &largest, &inputs->files, + hint_index, &hint_index); + } while (inputs->size() > old_size); - // Get the new range - GetRange(c->inputs_[0].files, &smallest, &largest); + // we started off with inputs non-empty and the previous loop only grew + // inputs. thus, inputs should be non-empty here + assert(!inputs->empty()); // If, after the expansion, there are files that are already under // compaction, then we must drop/cancel this compaction. - int parent_index = -1; - if (c->inputs_[0].empty()) { + if (FilesInCompaction(inputs->files)) { Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log, - "[%s] ExpandWhileOverlapping() failure because zero input files", + "[%s] ExpandWhileOverlapping() failure because some of the necessary" + " compaction input files are currently being compacted.", cf_name.c_str()); - } - if (c->inputs_[0].empty() || FilesInCompaction(c->inputs_[0].files) || - (c->level() != c->output_level() && - RangeInCompaction(vstorage, &smallest, &largest, c->output_level(), - &parent_index))) { - c->inputs_[0].clear(); - c->inputs_[c->num_input_levels() - 1].clear(); - if (!c->inputs_[0].empty()) { - Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log, - "[%s] ExpandWhileOverlapping() failure because some of the necessary" - " compaction input files are currently being compacted.", - c->column_family_data()->GetName().c_str()); - } return false; } return true; @@ -185,38 +165,23 @@ bool CompactionPicker::FilesInCompaction( } Compaction* CompactionPicker::FormCompaction( - const CompactionOptions& compact_options, - const autovector& input_files, - int output_level, VersionStorageInfo* vstorage, - const MutableCFOptions& mutable_cf_options) const { + const CompactionOptions& compact_options, + const std::vector& input_files, int output_level, + VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options, + uint32_t output_path_id) const { uint64_t max_grandparent_overlap_bytes = output_level + 1 < vstorage->num_levels() ? mutable_cf_options.MaxGrandParentOverlapBytes(output_level + 1) : std::numeric_limits::max(); assert(input_files.size()); - auto c = new Compaction(vstorage, input_files, - input_files[0].level, output_level, - max_grandparent_overlap_bytes, - compact_options, false); - c->mutable_cf_options_ = mutable_cf_options; - c->MarkFilesBeingCompacted(true); - - // TODO(yhchiang): complete the SetBottomMostLevel as follows - // If there is no any key of the range in DB that is older than the - // range to compact, it is bottom most. For leveled compaction, - // if number-of_level-1 is empty, and output is going to number-of_level-2, - // it is also bottom-most. On the other hand, if number of level=1 ( - // something like universal), the compaction is only "bottom-most" if - // the oldest file is involved. - c->SetupBottomMostLevel( - vstorage, - (output_level == vstorage->num_levels() - 1), - (output_level == 0)); - return c; + return new Compaction(vstorage, mutable_cf_options, input_files, output_level, + compact_options.output_file_size_limit, + max_grandparent_overlap_bytes, output_path_id, + compact_options.compression, /* grandparents */ {}); } Status CompactionPicker::GetCompactionInputsFromFileNumbers( - autovector* input_files, + std::vector* input_files, std::unordered_set* input_set, const VersionStorageInfo* vstorage, const CompactionOptions& compact_options) const { @@ -226,7 +191,7 @@ Status CompactionPicker::GetCompactionInputsFromFileNumbers( } assert(input_files); - autovector matched_input_files; + std::vector matched_input_files; matched_input_files.resize(vstorage->num_levels()); int first_non_empty_level = -1; int last_non_empty_level = -1; @@ -286,88 +251,100 @@ bool CompactionPicker::RangeInCompaction(VersionStorageInfo* vstorage, // Will also attempt to expand "start level" if that doesn't expand // "output level" or cause "level" to include a file for compaction that has an // overlapping user-key with another file. -void CompactionPicker::SetupOtherInputs( +// REQUIRES: input_level and output_level are different +// REQUIRES: inputs->empty() == false +// Returns false if files on parent level are currently in compaction, which +// means that we can't compact them +bool CompactionPicker::SetupOtherInputs( const std::string& cf_name, const MutableCFOptions& mutable_cf_options, - VersionStorageInfo* vstorage, Compaction* c) { - // If inputs are empty, then there is nothing to expand. - // If both input and output levels are the same, no need to consider - // files at level "level+1" - if (c->inputs_[0].empty() || c->level() == c->output_level()) { - return; - } + VersionStorageInfo* vstorage, CompactionInputFiles* inputs, + CompactionInputFiles* output_level_inputs, int* parent_index, + int base_index) { + assert(!inputs->empty()); + assert(output_level_inputs->empty()); + const int input_level = inputs->level; + const int output_level = output_level_inputs->level; + assert(input_level != output_level); // For now, we only support merging two levels, start level and output level. // We need to assert other levels are empty. - for (int l = c->start_level() + 1; l < c->output_level(); l++) { + for (int l = input_level + 1; l < output_level; l++) { assert(vstorage->NumLevelFiles(l) == 0); } - const int level = c->level(); InternalKey smallest, largest; // Get the range one last time. - GetRange(c->inputs_[0].files, &smallest, &largest); + GetRange(inputs->files, &smallest, &largest); // Populate the set of next-level files (inputs_GetOutputLevelInputs()) to // include in compaction - vstorage->GetOverlappingInputs(c->output_level(), &smallest, &largest, - &c->inputs_[c->num_input_levels() - 1].files, - c->parent_index_, &c->parent_index_); + vstorage->GetOverlappingInputs(output_level, &smallest, &largest, + &output_level_inputs->files, *parent_index, + parent_index); - // Get entire range covered by compaction - InternalKey all_start, all_limit; - GetRange(c->inputs_[0].files, c->inputs_[c->num_input_levels() - 1].files, - &all_start, &all_limit); + if (FilesInCompaction(output_level_inputs->files)) { + return false; + } // See if we can further grow the number of inputs in "level" without // changing the number of "level+1" files we pick up. We also choose NOT // to expand if this would cause "level" to include some entries for some // user key, while excluding other entries for the same user key. This // can happen when one user key spans multiple files. - if (!c->inputs(c->num_input_levels() - 1)->empty()) { + if (!output_level_inputs->empty()) { std::vector expanded0; - vstorage->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0, - c->base_index_, nullptr); - const uint64_t inputs0_size = TotalCompensatedFileSize(c->inputs_[0].files); + // Get entire range covered by compaction + InternalKey all_start, all_limit; + GetRange(inputs->files, output_level_inputs->files, &all_start, &all_limit); + + vstorage->GetOverlappingInputs(input_level, &all_start, &all_limit, + &expanded0, base_index, nullptr); + const uint64_t inputs0_size = TotalCompensatedFileSize(inputs->files); const uint64_t inputs1_size = - TotalCompensatedFileSize(c->inputs_[c->num_input_levels() - 1].files); + TotalCompensatedFileSize(output_level_inputs->files); const uint64_t expanded0_size = TotalCompensatedFileSize(expanded0); - uint64_t limit = mutable_cf_options.ExpandedCompactionByteSizeLimit(level); - if (expanded0.size() > c->inputs_[0].size() && + uint64_t limit = + mutable_cf_options.ExpandedCompactionByteSizeLimit(input_level); + if (expanded0.size() > inputs->size() && inputs1_size + expanded0_size < limit && !FilesInCompaction(expanded0) && - !vstorage->HasOverlappingUserKey(&expanded0, level)) { + !vstorage->HasOverlappingUserKey(&expanded0, input_level)) { InternalKey new_start, new_limit; GetRange(expanded0, &new_start, &new_limit); std::vector expanded1; - vstorage->GetOverlappingInputs(c->output_level(), &new_start, &new_limit, - &expanded1, c->parent_index_, - &c->parent_index_); - if (expanded1.size() == c->inputs(c->num_input_levels() - 1)->size() && + vstorage->GetOverlappingInputs(output_level, &new_start, &new_limit, + &expanded1, *parent_index, parent_index); + if (expanded1.size() == output_level_inputs->size() && !FilesInCompaction(expanded1)) { Log(InfoLogLevel::INFO_LEVEL, ioptions_.info_log, "[%s] Expanding@%d %zu+%zu (%" PRIu64 "+%" PRIu64 " bytes) to %zu+%zu (%" PRIu64 "+%" PRIu64 "bytes)\n", - cf_name.c_str(), level, c->inputs_[0].size(), - c->inputs(c->num_input_levels() - 1)->size(), inputs0_size, - inputs1_size, expanded0.size(), expanded1.size(), expanded0_size, - inputs1_size); + cf_name.c_str(), input_level, inputs->size(), + output_level_inputs->size(), inputs0_size, inputs1_size, + expanded0.size(), expanded1.size(), expanded0_size, inputs1_size); smallest = new_start; largest = new_limit; - c->inputs_[0].files = expanded0; - c->inputs_[c->num_input_levels() - 1].files = expanded1; - GetRange(c->inputs_[0].files, - c->inputs_[c->num_input_levels() - 1].files, &all_start, - &all_limit); + inputs->files = expanded0; + output_level_inputs->files = expanded1; } } } + return true; +} + +void CompactionPicker::GetGrandparents( + VersionStorageInfo* vstorage, const CompactionInputFiles& inputs, + const CompactionInputFiles& output_level_inputs, + std::vector* grandparents) { + InternalKey start, limit; + GetRange(inputs.files, output_level_inputs.files, &start, &limit); // Compute the set of grandparent files that overlap this compaction // (parent == level+1; grandparent == level+2) - if (c->output_level() + 1 < NumberLevels()) { - vstorage->GetOverlappingInputs(c->output_level() + 1, &all_start, - &all_limit, &c->grandparents_); + if (output_level_inputs.level + 1 < NumberLevels()) { + vstorage->GetOverlappingInputs(output_level_inputs.level + 1, &start, + &limit, grandparents); } } @@ -379,7 +356,8 @@ Compaction* CompactionPicker::CompactRange( // CompactionPickerFIFO has its own implementation of compact range assert(ioptions_.compaction_style != kCompactionStyleFIFO); - std::vector inputs; + CompactionInputFiles inputs; + inputs.level = input_level; bool covering_the_whole_range = true; // All files are 'overlapping' in universal style compaction. @@ -389,7 +367,7 @@ Compaction* CompactionPicker::CompactRange( end = nullptr; } - vstorage->GetOverlappingInputs(input_level, begin, end, &inputs); + vstorage->GetOverlappingInputs(input_level, begin, end, &inputs.files); if (inputs.empty()) { return nullptr; } @@ -408,46 +386,51 @@ Compaction* CompactionPicker::CompactRange( if (total >= limit) { **compaction_end = inputs[i + 1]->smallest; covering_the_whole_range = false; - inputs.resize(i + 1); + inputs.files.resize(i + 1); break; } } } assert(output_path_id < static_cast(ioptions_.db_paths.size())); - Compaction* c = new Compaction( - vstorage->num_levels(), input_level, output_level, - mutable_cf_options.MaxFileSizeForLevel(output_level), - mutable_cf_options.MaxGrandParentOverlapBytes(input_level), - output_path_id, - GetCompressionType(ioptions_, output_level, vstorage->base_level())); - c->inputs_[0].files = inputs; - if (ExpandWhileOverlapping(cf_name, vstorage, c) == false) { - delete c; - Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log, - "[%s] Could not compact due to expansion failure.\n", cf_name.c_str()); + if (ExpandWhileOverlapping(cf_name, vstorage, &inputs) == false) { + // manual compaction is currently single-threaded, so it should never + // happen that ExpandWhileOverlapping fails + assert(false); return nullptr; } - SetupOtherInputs(cf_name, mutable_cf_options, vstorage, c); - if (covering_the_whole_range) { *compaction_end = nullptr; } - // These files that are to be manaully compacted do not trample - // upon other files because manual compactions are processed when - // the system has a max of 1 background compaction thread. - c->MarkFilesBeingCompacted(true); - - // Is this compaction creating a file at the bottommost level - c->SetupBottomMostLevel( - vstorage, true, ioptions_.compaction_style == kCompactionStyleUniversal); + CompactionInputFiles output_level_inputs; + output_level_inputs.level = output_level; + if (input_level != output_level) { + int parent_index = -1; + if (!SetupOtherInputs(cf_name, mutable_cf_options, vstorage, &inputs, + &output_level_inputs, &parent_index, -1)) { + // manual compaction is currently single-threaded, so it should never + // happen that SetupOtherInputs fails + assert(false); + return nullptr; + } + } - c->is_manual_compaction_ = true; - c->mutable_cf_options_ = mutable_cf_options; + std::vector compaction_inputs({inputs}); + if (!output_level_inputs.empty()) { + compaction_inputs.push_back(output_level_inputs); + } - return c; + std::vector grandparents; + GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents); + return new Compaction( + vstorage, mutable_cf_options, std::move(compaction_inputs), output_level, + mutable_cf_options.MaxFileSizeForLevel(output_level), + mutable_cf_options.MaxGrandParentOverlapBytes(input_level), + output_path_id, + GetCompressionType(ioptions_, output_level, vstorage->base_level()), + std::move(grandparents), /* is manual compaction */ true); } #ifndef ROCKSDB_LITE @@ -694,63 +677,80 @@ bool LevelCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage) Compaction* LevelCompactionPicker::PickCompaction( const std::string& cf_name, const MutableCFOptions& mutable_cf_options, VersionStorageInfo* vstorage, LogBuffer* log_buffer) { - Compaction* c = nullptr; int level = -1; + int output_level = -1; + int parent_index = -1; + int base_index = -1; + CompactionInputFiles inputs; + double score; // Find the compactions by size on all levels. for (int i = 0; i < NumberLevels() - 1; i++) { - double score = vstorage->CompactionScore(i); + score = vstorage->CompactionScore(i); level = vstorage->CompactionScoreLevel(i); assert(i == 0 || score <= vstorage->CompactionScore(i - 1)); - if ((score >= 1)) { - c = PickCompactionBySize(mutable_cf_options, vstorage, level, score); - if (c == nullptr || - ExpandWhileOverlapping(cf_name, vstorage, c) == false) { - delete c; - c = nullptr; - } else { + if (score >= 1) { + output_level = (level == 0) ? vstorage->base_level() : level + 1; + if (PickCompactionBySize(vstorage, level, output_level, &inputs, + &parent_index, &base_index) && + ExpandWhileOverlapping(cf_name, vstorage, &inputs)) { + // found the compaction! break; } } } - if (c == nullptr) { + if (inputs.empty()) { return nullptr; } + assert(level >= 0 && output_level >= 0); // Two level 0 compaction won't run at the same time, so don't need to worry // about files on level 0 being compacted. if (level == 0) { assert(level0_compactions_in_progress_.empty()); InternalKey smallest, largest; - GetRange(c->inputs_[0].files, &smallest, &largest); + GetRange(inputs.files, &smallest, &largest); // Note that the next call will discard the file we placed in // c->inputs_[0] earlier and replace it with an overlapping set // which will include the picked file. - c->inputs_[0].clear(); - vstorage->GetOverlappingInputs(0, &smallest, &largest, - &c->inputs_[0].files); + inputs.files.clear(); + vstorage->GetOverlappingInputs(0, &smallest, &largest, &inputs.files); // If we include more L0 files in the same compaction run it can // cause the 'smallest' and 'largest' key to get extended to a // larger range. So, re-invoke GetRange to get the new key range - GetRange(c->inputs_[0].files, &smallest, &largest); - if (RangeInCompaction(vstorage, &smallest, &largest, c->output_level(), - &c->parent_index_)) { - delete c; + GetRange(inputs.files, &smallest, &largest); + if (RangeInCompaction(vstorage, &smallest, &largest, output_level, + &parent_index)) { return nullptr; } - assert(!c->inputs_[0].empty()); + assert(!inputs.files.empty()); } // Setup input files from output level - SetupOtherInputs(cf_name, mutable_cf_options, vstorage, c); + CompactionInputFiles output_level_inputs; + output_level_inputs.level = output_level; + if (!SetupOtherInputs(cf_name, mutable_cf_options, vstorage, &inputs, + &output_level_inputs, &parent_index, base_index)) { + return nullptr; + } - // mark all the files that are being compacted - c->MarkFilesBeingCompacted(true); + std::vector compaction_inputs({inputs}); + if (!output_level_inputs.empty()) { + compaction_inputs.push_back(output_level_inputs); + } - // Is this compaction creating a file at the bottommost level - c->SetupBottomMostLevel(vstorage, false, false); + std::vector grandparents; + GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents); + auto c = new Compaction( + vstorage, mutable_cf_options, std::move(compaction_inputs), output_level, + mutable_cf_options.MaxFileSizeForLevel(output_level), + mutable_cf_options.MaxGrandParentOverlapBytes(level), + GetPathId(ioptions_, mutable_cf_options, output_level), + GetCompressionType(ioptions_, output_level, vstorage->base_level()), + std::move(grandparents), + /* is manual */ false, score); // If it's level 0 compaction, make sure we don't execute any other level 0 // compactions in parallel @@ -758,8 +758,6 @@ Compaction* LevelCompactionPicker::PickCompaction( level0_compactions_in_progress_.insert(c); } - c->mutable_cf_options_ = mutable_cf_options; - // Creating a compaction influences the compaction score because the score // takes running compactions into account (by skipping files that are already // being compacted). Since we just changed compaction score, we recalculate it @@ -811,11 +809,11 @@ uint32_t LevelCompactionPicker::GetPathId( return p; } -Compaction* LevelCompactionPicker::PickCompactionBySize( - const MutableCFOptions& mutable_cf_options, VersionStorageInfo* vstorage, - int level, double score) { - Compaction* c = nullptr; - +bool LevelCompactionPicker::PickCompactionBySize(VersionStorageInfo* vstorage, + int level, int output_level, + CompactionInputFiles* inputs, + int* parent_index, + int* base_index) { // level 0 files are overlapping. So we cannot pick more // than one concurrent compactions at this level. This // could be made better by looking at key-ranges that are @@ -825,21 +823,6 @@ Compaction* LevelCompactionPicker::PickCompactionBySize( } assert(level >= 0); - int output_level; - if (level == 0) { - output_level = vstorage->base_level(); - } else { - output_level = level + 1; - } - assert(output_level < NumberLevels()); - - c = new Compaction( - vstorage->num_levels(), level, output_level, - mutable_cf_options.MaxFileSizeForLevel(output_level), - mutable_cf_options.MaxGrandParentOverlapBytes(level), - GetPathId(ioptions_, mutable_cf_options, output_level), - GetCompressionType(ioptions_, output_level, vstorage->base_level())); - c->score_ = score; // Pick the largest file in this level that is not already // being compacted @@ -852,7 +835,7 @@ Compaction* LevelCompactionPicker::PickCompactionBySize( for (unsigned int i = vstorage->NextCompactionIndex(level); i < file_size.size(); i++) { int index = file_size[i]; - FileMetaData* f = level_files[index]; + auto* f = level_files[index]; assert((i == file_size.size() - 1) || (i >= VersionStorageInfo::kNumberFilesToSort - 1) || @@ -872,26 +855,21 @@ Compaction* LevelCompactionPicker::PickCompactionBySize( // Do not pick this file if its parents at level+1 are being compacted. // Maybe we can avoid redoing this work in SetupOtherInputs - int parent_index = -1; - if (RangeInCompaction(vstorage, &f->smallest, &f->largest, - c->output_level(), &parent_index)) { + *parent_index = -1; + if (RangeInCompaction(vstorage, &f->smallest, &f->largest, output_level, + parent_index)) { continue; } - c->inputs_[0].files.push_back(f); - c->base_index_ = index; - c->parent_index_ = parent_index; + inputs->files.push_back(f); + inputs->level = level; + *base_index = index; break; } - if (c->inputs_[0].empty()) { - delete c; - c = nullptr; - } - // store where to start the iteration in the next call to PickCompaction vstorage->SetNextCompactionIndex(level, nextIndex); - return c; + return inputs->size() > 0; } #ifndef ROCKSDB_LITE @@ -1061,8 +1039,7 @@ Compaction* UniversalCompactionPicker::PickCompaction( size_t level_index = 0U; if (c->start_level() == 0) { - for (unsigned int i = 0; i < c->inputs_[0].size(); i++) { - FileMetaData* f = c->inputs_[0][i]; + for (auto f : *c->inputs(0)) { assert(f->smallest_seqno <= f->largest_seqno); if (is_first) { is_first = false; @@ -1088,43 +1065,12 @@ Compaction* UniversalCompactionPicker::PickCompaction( } } #endif - auto& last_sr = sorted_runs.back(); - if (c->output_level() < last_sr.level) { - // If it doesn't compact to the last level that has file, it's not the last - // bottom most. - c->bottommost_level_ = false; - } else if (last_sr.level == 0) { - // All files are level 0. Then the compaction is bottom most iff it includes - // the last file. - c->bottommost_level_ = c->inputs_[kLevel0].files.back() == last_sr.file; - } else { - // In this case, it's not level 0 only and the compaction includes the last - // level having files. It is bottom most. - c->bottommost_level_ = true; - } - // update statistics - MeasureTime(ioptions_.statistics, - NUM_FILES_IN_SINGLE_COMPACTION, c->inputs_[kLevel0].size()); - - // mark all the files that are being compacted - c->MarkFilesBeingCompacted(true); + MeasureTime(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION, + c->inputs(0)->size()); level0_compactions_in_progress_.insert(c); - // Record whether this compaction includes all sst files. - // For now, it is only relevant in universal compaction mode. - int num_files_in_compaction = 0; - int total_num_files = 0; - for (int level = 0; level < vstorage->num_levels(); level++) { - total_num_files += vstorage->NumLevelFiles(level); - } - for (size_t i = 0; i < c->num_input_levels(); i++) { - num_files_in_compaction += c->num_input_files(i); - } - c->is_full_compaction_ = (num_files_in_compaction == total_num_files); - - c->mutable_cf_options_ = mutable_cf_options; return c; } @@ -1309,19 +1255,17 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( output_level = sorted_runs[first_index_after].level - 1; } - Compaction* c = new Compaction( - vstorage->num_levels(), start_level, output_level, - mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id, - GetCompressionType(ioptions_, start_level, 1, enable_compression)); - c->score_ = score; - + std::vector inputs(vstorage->num_levels()); + for (size_t i = 0; i < inputs.size(); ++i) { + inputs[i].level = start_level + static_cast(i); + } for (unsigned int i = start_index; i < first_index_after; i++) { auto& picking_sr = sorted_runs[i]; if (picking_sr.level == 0) { FileMetaData* picking_file = picking_sr.file; - c->inputs_[0].files.push_back(picking_file); + inputs[0].files.push_back(picking_file); } else { - auto& files = c->inputs_[picking_sr.level - start_level].files; + auto& files = inputs[picking_sr.level - start_level].files; for (auto* f : vstorage->LevelFiles(picking_sr.level)) { files.push_back(f); } @@ -1331,7 +1275,12 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( LogToBuffer(log_buffer, "[%s] Universal: Picking %s", cf_name.c_str(), file_num_buf); } - return c; + + return new Compaction( + vstorage, mutable_cf_options, std::move(inputs), output_level, + mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id, + GetCompressionType(ioptions_, start_level, 1, enable_compression), + /* grandparents */ {}, /* is manual */ false, score); } // Look at overall size amplification. If size amplification @@ -1426,21 +1375,18 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( uint32_t path_id = GetPathId(ioptions_, estimated_total_size); int start_level = sorted_runs[start_index].level; - // create a compaction request + std::vector inputs(vstorage->num_levels()); + for (size_t i = 0; i < inputs.size(); ++i) { + inputs[i].level = start_level + static_cast(i); + } // We always compact all the files, so always compress. - Compaction* c = new Compaction( - vstorage->num_levels(), start_level, vstorage->num_levels() - 1, - mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1), - LLONG_MAX, path_id, - GetCompressionType(ioptions_, vstorage->num_levels() - 1, 1)); - c->score_ = score; for (unsigned int loop = start_index; loop < sorted_runs.size(); loop++) { auto& picking_sr = sorted_runs[loop]; if (picking_sr.level == 0) { FileMetaData* f = picking_sr.file; - c->inputs_[0].files.push_back(f); + inputs[0].files.push_back(f); } else { - auto& files = c->inputs_[picking_sr.level - start_level].files; + auto& files = inputs[picking_sr.level - start_level].files; for (auto* f : vstorage->LevelFiles(picking_sr.level)) { files.push_back(f); } @@ -1450,7 +1396,14 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( LogToBuffer(log_buffer, "[%s] Universal: size amp picking %s", cf_name.c_str(), file_num_buf); } - return c; + + return new Compaction( + vstorage, mutable_cf_options, std::move(inputs), + vstorage->num_levels() - 1, + mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1), + /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id, + GetCompressionType(ioptions_, vstorage->num_levels() - 1, 1), + /* grandparents */ {}, /* is manual */ false, score); } bool FIFOCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage) @@ -1489,13 +1442,14 @@ Compaction* FIFOCompactionPicker::PickCompaction( return nullptr; } - Compaction* c = new Compaction(1, 0, 0, 0, 0, 0, kNoCompression, false, - true /* is deletion compaction */); + std::vector inputs; + inputs.emplace_back(); + inputs[0].level = 0; // delete old files (FIFO) for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) { auto f = *ritr; total_size -= f->compensated_file_size; - c->inputs_[0].files.push_back(f); + inputs[0].files.push_back(f); char tmp_fsize[16]; AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize)); LogToBuffer(log_buffer, "[%s] FIFO compaction: picking file %" PRIu64 @@ -1505,10 +1459,11 @@ Compaction* FIFOCompactionPicker::PickCompaction( break; } } - - c->MarkFilesBeingCompacted(true); + Compaction* c = new Compaction( + vstorage, mutable_cf_options, std::move(inputs), 0, 0, 0, 0, + kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0), + /* is deletion compaction */ true); level0_compactions_in_progress_.insert(c); - c->mutable_cf_options_ = mutable_cf_options; return c; } @@ -1523,10 +1478,6 @@ Compaction* FIFOCompactionPicker::CompactRange( LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.info_log); Compaction* c = PickCompaction(cf_name, mutable_cf_options, vstorage, &log_buffer); - if (c != nullptr) { - assert(output_path_id < static_cast(ioptions_.db_paths.size())); - c->output_path_id_ = output_path_id; - } log_buffer.FlushBufferToLog(); return c; } diff --git a/db/compaction_picker.h b/db/compaction_picker.h index 43ccfcc9a..c3f1a265a 100644 --- a/db/compaction_picker.h +++ b/db/compaction_picker.h @@ -97,14 +97,14 @@ class CompactionPicker { // Takes a list of CompactionInputFiles and returns a Compaction object. Compaction* FormCompaction( const CompactionOptions& compact_options, - const autovector& input_files, - int output_level, VersionStorageInfo* vstorage, - const MutableCFOptions& mutable_cf_options) const; + const std::vector& input_files, int output_level, + VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options, + uint32_t output_path_id) const; // Converts a set of compaction input file numbers into // a list of CompactionInputFiles. Status GetCompactionInputsFromFileNumbers( - autovector* input_files, + std::vector* input_files, std::unordered_set* input_set, const VersionStorageInfo* vstorage, const CompactionOptions& compact_options) const; @@ -136,16 +136,25 @@ class CompactionPicker { // // Will return false if it is impossible to apply this compaction. bool ExpandWhileOverlapping(const std::string& cf_name, - VersionStorageInfo* vstorage, Compaction* c); + VersionStorageInfo* vstorage, + CompactionInputFiles* inputs); // Returns true if any one of the parent files are being compacted bool RangeInCompaction(VersionStorageInfo* vstorage, const InternalKey* smallest, const InternalKey* largest, int level, int* index); - void SetupOtherInputs(const std::string& cf_name, + bool SetupOtherInputs(const std::string& cf_name, const MutableCFOptions& mutable_cf_options, - VersionStorageInfo* vstorage, Compaction* c); + VersionStorageInfo* vstorage, + CompactionInputFiles* inputs, + CompactionInputFiles* output_level_inputs, + int* parent_index, int base_index); + + void GetGrandparents(VersionStorageInfo* vstorage, + const CompactionInputFiles& inputs, + const CompactionInputFiles& output_level_inputs, + std::vector* grandparents); const ImmutableCFOptions& ioptions_; @@ -190,13 +199,14 @@ class LevelCompactionPicker : public CompactionPicker { int level); private: - // For the specfied level, pick a compaction. - // Returns nullptr if there is no compaction to be done. + // For the specfied level, pick a file that we want to compact. + // Returns false if there is no file to compact. + // If it returns true, inputs->files.size() will be exactly one. // If level is 0 and there is already a compaction on that level, this - // function will return nullptr. - Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options, - VersionStorageInfo* vstorage, int level, - double score); + // function will return false. + bool PickCompactionBySize(VersionStorageInfo* vstorage, int level, + int output_level, CompactionInputFiles* inputs, + int* parent_index, int* base_index); }; #ifndef ROCKSDB_LITE diff --git a/db/compaction_picker_test.cc b/db/compaction_picker_test.cc index 2d6ac0068..d3789dcfb 100644 --- a/db/compaction_picker_test.cc +++ b/db/compaction_picker_test.cc @@ -229,7 +229,7 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic) { ASSERT_EQ(2U, compaction->num_input_files(0)); ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); - ASSERT_EQ(num_levels, static_cast(compaction->num_input_levels())); + ASSERT_EQ(1, static_cast(compaction->num_input_levels())); ASSERT_EQ(num_levels - 1, compaction->output_level()); } @@ -253,7 +253,7 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic2) { ASSERT_EQ(2U, compaction->num_input_files(0)); ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); - ASSERT_EQ(num_levels - 1, static_cast(compaction->num_input_levels())); + ASSERT_EQ(1, static_cast(compaction->num_input_levels())); ASSERT_EQ(num_levels - 2, compaction->output_level()); } @@ -278,7 +278,7 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic3) { ASSERT_EQ(2U, compaction->num_input_files(0)); ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); - ASSERT_EQ(num_levels - 2, static_cast(compaction->num_input_levels())); + ASSERT_EQ(1, static_cast(compaction->num_input_levels())); ASSERT_EQ(num_levels - 3, compaction->output_level()); } @@ -306,10 +306,11 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic4) { ASSERT_EQ(2U, compaction->num_input_files(0)); ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); - ASSERT_EQ(2U, compaction->num_input_files(num_levels - 3)); - ASSERT_EQ(5U, compaction->input(num_levels - 3, 0)->fd.GetNumber()); - ASSERT_EQ(6U, compaction->input(num_levels - 3, 1)->fd.GetNumber()); - ASSERT_EQ(num_levels - 2, static_cast(compaction->num_input_levels())); + ASSERT_EQ(2U, compaction->num_input_files(1)); + ASSERT_EQ(num_levels - 3, compaction->level(1)); + ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(2, static_cast(compaction->num_input_levels())); ASSERT_EQ(num_levels - 3, compaction->output_level()); } diff --git a/db/db_impl.cc b/db/db_impl.cc index f0fba4950..451ec225f 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1399,7 +1399,7 @@ Status DBImpl::CompactFilesImpl( return s; } - autovector input_files; + std::vector input_files; s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers( &input_files, &input_set, version->storage_info(), compact_options); if (!s.ok()) { @@ -1420,12 +1420,10 @@ Status DBImpl::CompactFilesImpl( unique_ptr c; assert(cfd->compaction_picker()); c.reset(cfd->compaction_picker()->FormCompaction( - compact_options, input_files, - output_level, version->storage_info(), - *cfd->GetLatestMutableCFOptions())); + compact_options, input_files, output_level, version->storage_info(), + *cfd->GetLatestMutableCFOptions(), output_path_id)); assert(c); c->SetInputVersion(version); - c->SetOutputPathId(static_cast(output_path_id)); // deletion compaction currently not allowed in CompactFiles. assert(!c->IsDeletionCompaction()); diff --git a/db/db_test.cc b/db/db_test.cc index b69548720..dd9477b55 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -12457,6 +12457,7 @@ TEST_F(DBTest, CompressLevelCompaction) { } // namespace rocksdb int main(int argc, char** argv) { + rocksdb::port::InstallStackTraceHandler(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); }