Fix caching of compaction picker's next index

Summary:
The previous implementation of caching `file_size` index made no sense. It only remembered the original span of locked files starting from beginning of `file_size`. We should remember the index after all compactions that have been considered but rejected. This will reduce the work we do while holding the db mutex.
Closes https://github.com/facebook/rocksdb/pull/2624

Differential Revision: D5468152

Pulled By: ajkr

fbshipit-source-id: ab92a4bffe76f9f174d861bb5812b974d1013400
main
Andrew Kryczka 7 years ago committed by Facebook Github Bot
parent 72502cf227
commit a34b2e388e
  1. 17
      db/compaction_picker.cc
  2. 43
      db/compaction_picker_test.cc

@ -1323,12 +1323,10 @@ bool LevelCompactionBuilder::PickFileToCompact() {
const std::vector<FileMetaData*>& level_files = const std::vector<FileMetaData*>& level_files =
vstorage_->LevelFiles(start_level_); vstorage_->LevelFiles(start_level_);
// record the first file that is not yet compacted unsigned int cmp_idx;
int nextIndex = -1; for (cmp_idx = vstorage_->NextCompactionIndex(start_level_);
cmp_idx < file_size.size(); cmp_idx++) {
for (unsigned int i = vstorage_->NextCompactionIndex(start_level_); int index = file_size[cmp_idx];
i < file_size.size(); i++) {
int index = file_size[i];
auto* f = level_files[index]; auto* f = level_files[index];
// do not pick a file to compact if it is being compacted // do not pick a file to compact if it is being compacted
@ -1337,11 +1335,6 @@ bool LevelCompactionBuilder::PickFileToCompact() {
continue; continue;
} }
// remember the startIndex for the next call to PickCompaction
if (nextIndex == -1) {
nextIndex = i;
}
start_level_inputs_.files.push_back(f); start_level_inputs_.files.push_back(f);
start_level_inputs_.level = start_level_; start_level_inputs_.level = start_level_;
if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_, if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
@ -1377,7 +1370,7 @@ bool LevelCompactionBuilder::PickFileToCompact() {
} }
// store where to start the iteration in the next call to PickCompaction // store where to start the iteration in the next call to PickCompaction
vstorage_->SetNextCompactionIndex(start_level_, nextIndex); vstorage_->SetNextCompactionIndex(start_level_, cmp_idx);
return start_level_inputs_.size() > 0; return start_level_inputs_.size() > 0;
} }

@ -1390,6 +1390,49 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
ASSERT_FALSE(compaction->IsTrivialMove()); ASSERT_FALSE(compaction->IsTrivialMove());
} }
TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.max_compaction_bytes = 100000000000u;
Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
"149" /* largest */, 1000000000U /* file_size */);
file_map_[1U].first->being_compacted = true;
Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
"199" /* largest */, 900000000U /* file_size */);
Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
"249" /* largest */, 800000000U /* file_size */);
Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
"299" /* largest */, 700000000U /* file_size */);
Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
"199" /* largest */, 1U /* file_size */);
file_map_[5U].first->being_compacted = true;
UpdateVersionStorageInfo();
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_levels());
ASSERT_EQ(1U, compaction->num_input_files(0));
ASSERT_EQ(0U, compaction->num_input_files(1));
ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
compaction.reset(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_levels());
ASSERT_EQ(1U, compaction->num_input_files(0));
ASSERT_EQ(0U, compaction->num_input_files(1));
ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
compaction.reset(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() == nullptr);
ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
}
} // namespace rocksdb } // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {

Loading…
Cancel
Save