From 81747f1be66f1426cff3f701a15b00d7b3945044 Mon Sep 17 00:00:00 2001 From: Yi Wu Date: Tue, 13 Sep 2016 21:11:59 -0700 Subject: [PATCH] Refactor MutableCFOptions Summary: * Change constructor of MutableCFOptions to depends only on ColumnFamilyOptions. * Move `max_subcompactions`, `compaction_options_fifo` and `compaction_pri` to ImmutableCFOptions to make it clear that they are immutable. Test Plan: existing unit tests. Reviewers: yhchiang, IslamAbdelRahman, sdong Reviewed By: sdong Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D63945 --- db/column_family.cc | 2 +- db/column_family_test.cc | 16 +++--------- db/compaction.cc | 4 ++- db/compaction.h | 10 +++++++- db/compaction_job_test.cc | 4 +-- db/compaction_picker.cc | 26 ++++++++++---------- db/compaction_picker_test.cc | 16 ++++++------ db/db_impl.cc | 8 +++--- db/db_impl_experimental.cc | 3 ++- db/memtable_list_test.cc | 41 ++++++++++++++----------------- db/repair.cc | 2 +- db/version_builder_test.cc | 6 ++--- db/version_set.cc | 15 ++++++----- db/version_set.h | 5 ++-- db/version_set_test.cc | 2 +- db/write_batch_test.cc | 5 ++-- java/rocksjni/write_batch_test.cc | 3 +-- table/sst_file_writer.cc | 2 +- table/table_test.cc | 15 ++++++----- util/cf_options.cc | 27 ++++++++++++++------ util/cf_options.h | 25 ++++++++++--------- 21 files changed, 124 insertions(+), 113 deletions(-) diff --git a/db/column_family.cc b/db/column_family.cc index d2dd72d94..54f59d8cb 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -350,7 +350,7 @@ ColumnFamilyData::ColumnFamilyData( options_(*db_options, SanitizeOptions(*db_options, &internal_comparator_, cf_options)), ioptions_(options_), - mutable_cf_options_(options_, ioptions_), + mutable_cf_options_(options_), write_buffer_manager_(write_buffer_manager), mem_(nullptr), imm_(options_.min_write_buffer_number_to_merge, diff --git a/db/column_family_test.cc b/db/column_family_test.cc index b4e11867d..d38deb6b6 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -2440,9 +2440,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) { VersionStorageInfo* vstorage = cfd->current()->storage_info(); - MutableCFOptions mutable_cf_options( - Options(db_options_, column_family_options_), - ImmutableCFOptions(Options(db_options_, column_family_options_))); + MutableCFOptions mutable_cf_options(column_family_options_); mutable_cf_options.level0_slowdown_writes_trigger = 20; mutable_cf_options.level0_stop_writes_trigger = 10000; @@ -2631,9 +2629,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) { VersionStorageInfo* vstorage = cfd->current()->storage_info(); - MutableCFOptions mutable_cf_options( - Options(db_options_, column_family_options_), - ImmutableCFOptions(Options(db_options_, column_family_options_))); + MutableCFOptions mutable_cf_options(column_family_options_); // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8 mutable_cf_options.level0_file_num_compaction_trigger = 4; @@ -2702,9 +2698,7 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) { static_cast(handles_[1])->cfd(); VersionStorageInfo* vstorage1 = cfd1->current()->storage_info(); - MutableCFOptions mutable_cf_options( - Options(db_options_, column_family_options_), - ImmutableCFOptions(Options(db_options_, column_family_options_))); + MutableCFOptions mutable_cf_options(column_family_options_); mutable_cf_options.level0_slowdown_writes_trigger = 20; mutable_cf_options.level0_stop_writes_trigger = 10000; mutable_cf_options.soft_pending_compaction_bytes_limit = 200; @@ -2786,9 +2780,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) { static_cast(handles_[1])->cfd(); VersionStorageInfo* vstorage1 = cfd1->current()->storage_info(); - MutableCFOptions mutable_cf_options( - Options(db_options_, column_family_options_), - ImmutableCFOptions(Options(db_options_, column_family_options_))); + MutableCFOptions mutable_cf_options(column_family_options_); // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8 mutable_cf_options.level0_file_num_compaction_trigger = 4; mutable_cf_options.level0_slowdown_writes_trigger = 36; diff --git a/db/compaction.cc b/db/compaction.cc index 3791cd522..d168cf75d 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -140,6 +140,7 @@ bool Compaction::IsFullCompaction( } Compaction::Compaction(VersionStorageInfo* vstorage, + const ImmutableCFOptions& _immutable_cf_options, const MutableCFOptions& _mutable_cf_options, std::vector _inputs, int _output_level, uint64_t _target_file_size, @@ -153,6 +154,7 @@ Compaction::Compaction(VersionStorageInfo* vstorage, output_level_(_output_level), max_output_file_size_(_target_file_size), max_compaction_bytes_(_max_compaction_bytes), + immutable_cf_options_(_immutable_cf_options), mutable_cf_options_(_mutable_cf_options), input_version_(nullptr), number_levels_(vstorage->num_levels()), @@ -427,7 +429,7 @@ bool Compaction::IsOutputLevelEmpty() const { } bool Compaction::ShouldFormSubcompactions() const { - if (mutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) { + if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) { return false; } if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { diff --git a/db/compaction.h b/db/compaction.h index a412ffbb6..058e4dab0 100644 --- a/db/compaction.h +++ b/db/compaction.h @@ -35,6 +35,7 @@ class CompactionFilter; class Compaction { public: Compaction(VersionStorageInfo* input_version, + const ImmutableCFOptions& immutable_cf_options, const MutableCFOptions& mutable_cf_options, std::vector inputs, int output_level, uint64_t target_file_size, uint64_t max_compaction_bytes, @@ -171,6 +172,12 @@ class Compaction { // How many total levels are there? int number_levels() const { return number_levels_; } + // Return the ImmutableCFOptions that should be used throughout the compaction + // procedure + const ImmutableCFOptions* immutable_cf_options() const { + return &immutable_cf_options_; + } + // Return the MutableCFOptions that should be used throughout the compaction // procedure const MutableCFOptions* mutable_cf_options() const { @@ -253,7 +260,8 @@ class Compaction { const int output_level_; // levels to which output files are stored uint64_t max_output_file_size_; uint64_t max_compaction_bytes_; - MutableCFOptions mutable_cf_options_; + const ImmutableCFOptions immutable_cf_options_; + const MutableCFOptions mutable_cf_options_; Version* input_version_; VersionEdit edit_; const int number_levels_; diff --git a/db/compaction_job_test.cc b/db/compaction_job_test.cc index d4af6c26e..10236a093 100644 --- a/db/compaction_job_test.cc +++ b/db/compaction_job_test.cc @@ -68,7 +68,7 @@ class CompactionJobTest : public testing::Test { CompactionJobTest() : env_(Env::Default()), dbname_(test::TmpDir() + "/compaction_job_test"), - mutable_cf_options_(Options(), ImmutableCFOptions(Options())), + mutable_cf_options_(Options()), table_cache_(NewLRUCache(50000, 16)), write_buffer_manager_(db_options_.db_write_buffer_size), versions_(new VersionSet(dbname_, &db_options_, env_options_, @@ -239,7 +239,7 @@ class CompactionJobTest : public testing::Test { num_input_files += level_files.size(); } - Compaction compaction(cfd->current()->storage_info(), + Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(), *cfd->GetLatestMutableCFOptions(), compaction_input_files, 1, 1024 * 1024, 10 * 1024 * 1024, 0, kNoCompression, {}, true); diff --git a/db/compaction_picker.cc b/db/compaction_picker.cc index fed261c00..67e4ec9de 100644 --- a/db/compaction_picker.cc +++ b/db/compaction_picker.cc @@ -275,8 +275,8 @@ Compaction* CompactionPicker::FormCompaction( return nullptr; } auto c = - new Compaction(vstorage, mutable_cf_options, input_files, output_level, - compact_options.output_file_size_limit, + new Compaction(vstorage, ioptions_, mutable_cf_options, input_files, + output_level, compact_options.output_file_size_limit, mutable_cf_options.max_compaction_bytes, output_path_id, compact_options.compression, /* grandparents */ {}, true); @@ -507,8 +507,8 @@ Compaction* CompactionPicker::CompactRange( } } Compaction* c = new Compaction( - vstorage, mutable_cf_options, std::move(inputs), output_level, - mutable_cf_options.MaxFileSizeForLevel(output_level), + vstorage, ioptions_, mutable_cf_options, std::move(inputs), + output_level, mutable_cf_options.MaxFileSizeForLevel(output_level), /* max_compaction_bytes */ LLONG_MAX, output_path_id, GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level, 1), @@ -607,8 +607,8 @@ Compaction* CompactionPicker::CompactRange( std::vector grandparents; GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents); Compaction* compaction = new Compaction( - vstorage, mutable_cf_options, std::move(compaction_inputs), output_level, - mutable_cf_options.MaxFileSizeForLevel(output_level), + vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs), + output_level, mutable_cf_options.MaxFileSizeForLevel(output_level), mutable_cf_options.max_compaction_bytes, output_path_id, GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level, vstorage->base_level()), @@ -623,7 +623,7 @@ Compaction* CompactionPicker::CompactRange( // takes running compactions into account (by skipping files that are already // being compacted). Since we just changed compaction score, we recalculate it // here - vstorage->ComputeCompactionScore(mutable_cf_options); + vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options); return compaction; } @@ -1012,8 +1012,8 @@ Compaction* LevelCompactionPicker::PickCompaction( std::vector grandparents; GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents); auto c = new Compaction( - vstorage, mutable_cf_options, std::move(compaction_inputs), output_level, - mutable_cf_options.MaxFileSizeForLevel(output_level), + vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs), + output_level, mutable_cf_options.MaxFileSizeForLevel(output_level), mutable_cf_options.max_compaction_bytes, GetPathId(ioptions_, mutable_cf_options, output_level), GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level, @@ -1031,7 +1031,7 @@ Compaction* LevelCompactionPicker::PickCompaction( // takes running compactions into account (by skipping files that are already // being compacted). Since we just changed compaction score, we recalculate it // here - vstorage->ComputeCompactionScore(mutable_cf_options); + vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options); TEST_SYNC_POINT_CALLBACK("LevelCompactionPicker::PickCompaction:Return", c); @@ -1643,7 +1643,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( compaction_reason = CompactionReason::kUniversalSizeRatio; } return new Compaction( - vstorage, mutable_cf_options, std::move(inputs), output_level, + vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level, mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id, GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level, 1, enable_compression), @@ -1767,7 +1767,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( } return new Compaction( - vstorage, mutable_cf_options, std::move(inputs), + vstorage, ioptions_, mutable_cf_options, std::move(inputs), vstorage->num_levels() - 1, mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1), /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id, @@ -1832,7 +1832,7 @@ Compaction* FIFOCompactionPicker::PickCompaction( } } Compaction* c = new Compaction( - vstorage, mutable_cf_options, std::move(inputs), 0, 0, 0, 0, + vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0, kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0), /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize); level0_compactions_in_progress_.insert(c); diff --git a/db/compaction_picker_test.cc b/db/compaction_picker_test.cc index 2501855d9..e23c8e87d 100644 --- a/db/compaction_picker_test.cc +++ b/db/compaction_picker_test.cc @@ -48,7 +48,7 @@ class CompactionPickerTest : public testing::Test { : ucmp_(BytewiseComparator()), icmp_(ucmp_), ioptions_(options_), - mutable_cf_options_(options_, ioptions_), + mutable_cf_options_(options_), level_compaction_picker(ioptions_, &icmp_), cf_name_("dummy"), log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_), @@ -115,11 +115,11 @@ class CompactionPickerTest : public testing::Test { void UpdateVersionStorageInfo() { vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_); - vstorage_->UpdateFilesByCompactionPri(mutable_cf_options_); + vstorage_->UpdateFilesByCompactionPri(ioptions_.compaction_pri); vstorage_->UpdateNumNonEmptyLevels(); vstorage_->GenerateFileIndexer(); vstorage_->GenerateLevelFilesBrief(); - vstorage_->ComputeCompactionScore(mutable_cf_options_); + vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_); vstorage_->GenerateLevel0NonOverlapping(); vstorage_->SetFinalized(); } @@ -350,10 +350,10 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic4) { TEST_F(CompactionPickerTest, LevelTriggerDynamic4) { int num_levels = ioptions_.num_levels; ioptions_.level_compaction_dynamic_level_bytes = true; + ioptions_.compaction_pri = kMinOverlappingRatio; mutable_cf_options_.level0_file_num_compaction_trigger = 2; mutable_cf_options_.max_bytes_for_level_base = 200; mutable_cf_options_.max_bytes_for_level_multiplier = 10; - mutable_cf_options_.compaction_pri = kMinOverlappingRatio; NewVersionStorage(num_levels, kCompactionStyleLevel); Add(0, 1U, "150", "200"); Add(num_levels - 1, 3U, "200", "250", 300U); @@ -489,10 +489,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; - mutable_cf_options_.compaction_pri = kMinOverlappingRatio; Add(2, 6U, "150", "179", 50000000U); Add(2, 7U, "180", "220", 50000000U); @@ -516,10 +516,10 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) { NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; - mutable_cf_options_.compaction_pri = kMinOverlappingRatio; Add(2, 6U, "150", "175", 60000000U); // Overlaps with file 26, 27, total size 521M @@ -546,9 +546,9 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) { NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; mutable_cf_options_.max_bytes_for_level_base = 10000000; mutable_cf_options_.max_bytes_for_level_multiplier = 10; - mutable_cf_options_.compaction_pri = kMinOverlappingRatio; // file 7 and 8 over lap with the same file, but file 8 is smaller so // it will be picked. @@ -598,7 +598,7 @@ TEST_F(CompactionPickerTest, ParentIndexResetBug) { // ranges (with different sequence numbers) in the input files. TEST_F(CompactionPickerTest, OverlappingUserKeys) { NewVersionStorage(6, kCompactionStyleLevel); - mutable_cf_options_.compaction_pri = kByCompensatedSize; + ioptions_.compaction_pri = kByCompensatedSize; Add(1, 1U, "100", "150", 1U); // Overlapping user keys diff --git a/db/db_impl.cc b/db/db_impl.cc index e2c26bf5e..2b2cbfad8 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -2263,7 +2263,8 @@ Status DBImpl::CompactFilesImpl( // takes running compactions into account (by skipping files that are already // being compacted). Since we just changed compaction score, we recalculate it // here. - version->storage_info()->ComputeCompactionScore(*c->mutable_cf_options()); + version->storage_info()->ComputeCompactionScore(*cfd->ioptions(), + *c->mutable_cf_options()); compaction_job.Prepare(); @@ -4037,9 +4038,8 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options, write_thread_.EnterUnbatched(&w, &mutex_); // LogAndApply will both write the creation in MANIFEST and create // ColumnFamilyData object - s = versions_->LogAndApply( - nullptr, MutableCFOptions(opt, ImmutableCFOptions(opt)), &edit, - &mutex_, directories_.GetDbDir(), false, &cf_options); + s = versions_->LogAndApply(nullptr, MutableCFOptions(opt), &edit, &mutex_, + directories_.GetDbDir(), false, &cf_options); if (s.ok()) { // If the column family was created successfully, we then persist diff --git a/db/db_impl_experimental.cc b/db/db_impl_experimental.cc index 90e034cd0..139b0d01d 100644 --- a/db/db_impl_experimental.cc +++ b/db/db_impl_experimental.cc @@ -49,7 +49,8 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family, } // Since we have some more files to compact, we should also recompute // compaction score - vstorage->ComputeCompactionScore(*cfd->GetLatestMutableCFOptions()); + vstorage->ComputeCompactionScore(*cfd->ioptions(), + *cfd->GetLatestMutableCFOptions()); SchedulePendingCompaction(cfd); MaybeScheduleFlushOrCompaction(); } diff --git a/db/memtable_list_test.cc b/db/memtable_list_test.cc index 42420b876..f9f73f976 100644 --- a/db/memtable_list_test.cc +++ b/db/memtable_list_test.cc @@ -127,9 +127,8 @@ TEST_F(MemTableListTest, GetTest) { ImmutableCFOptions ioptions(options); WriteBufferManager wb(options.db_write_buffer_size); - MemTable* mem = - new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb, - kMaxSequenceNumber); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber); mem->Ref(); // Write some keys to this memtable. @@ -164,9 +163,8 @@ TEST_F(MemTableListTest, GetTest) { // Create another memtable and write some keys to it WriteBufferManager wb2(options.db_write_buffer_size); - MemTable* mem2 = - new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb2, - kMaxSequenceNumber); + MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2, + kMaxSequenceNumber); mem2->Ref(); mem2->Add(++seq, kTypeDeletion, "key1", ""); @@ -229,9 +227,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) { ImmutableCFOptions ioptions(options); WriteBufferManager wb(options.db_write_buffer_size); - MemTable* mem = - new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb, - kMaxSequenceNumber); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber); mem->Ref(); // Write some keys to this memtable. @@ -272,8 +269,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) { list.PickMemtablesToFlush(&to_flush); ASSERT_EQ(1, to_flush.size()); - s = Mock_InstallMemtableFlushResults( - &list, MutableCFOptions(options, ioptions), to_flush, &to_delete); + s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options), + to_flush, &to_delete); ASSERT_OK(s); ASSERT_EQ(0, list.NumNotFlushed()); ASSERT_EQ(1, list.NumFlushed()); @@ -304,9 +301,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) { // Create another memtable and write some keys to it WriteBufferManager wb2(options.db_write_buffer_size); - MemTable* mem2 = - new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb2, - kMaxSequenceNumber); + MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2, + kMaxSequenceNumber); mem2->Ref(); mem2->Add(++seq, kTypeDeletion, "key1", ""); @@ -321,8 +317,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) { ASSERT_EQ(1, to_flush.size()); // Flush second memtable - s = Mock_InstallMemtableFlushResults( - &list, MutableCFOptions(options, ioptions), to_flush, &to_delete); + s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options), + to_flush, &to_delete); ASSERT_OK(s); ASSERT_EQ(0, list.NumNotFlushed()); ASSERT_EQ(2, list.NumFlushed()); @@ -330,9 +326,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) { // Add a third memtable to push the first memtable out of the history WriteBufferManager wb3(options.db_write_buffer_size); - MemTable* mem3 = - new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb3, - kMaxSequenceNumber); + MemTable* mem3 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb3, + kMaxSequenceNumber); mem3->Ref(); list.Add(mem3, &to_delete); ASSERT_EQ(1, list.NumNotFlushed()); @@ -401,7 +396,7 @@ TEST_F(MemTableListTest, FlushPendingTest) { // Create some MemTables std::vector tables; - MutableCFOptions mutable_cf_options(options, ioptions); + MutableCFOptions mutable_cf_options(options); for (int i = 0; i < num_tables; i++) { MemTable* mem = new MemTable(cmp, ioptions, mutable_cf_options, &wb, kMaxSequenceNumber); @@ -539,8 +534,8 @@ TEST_F(MemTableListTest, FlushPendingTest) { ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); // Flush the 4 memtables that were picked in to_flush - s = Mock_InstallMemtableFlushResults( - &list, MutableCFOptions(options, ioptions), to_flush, &to_delete); + s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options), + to_flush, &to_delete); ASSERT_OK(s); // Note: now to_flush contains tables[0,1,2,4]. to_flush2 contains @@ -560,7 +555,7 @@ TEST_F(MemTableListTest, FlushPendingTest) { // Flush the 1 memtable that was picked in to_flush2 s = MemTableListTest::Mock_InstallMemtableFlushResults( - &list, MutableCFOptions(options, ioptions), to_flush2, &to_delete); + &list, MutableCFOptions(options), to_flush2, &to_delete); ASSERT_OK(s); // This will actually install 2 tables. The 1 we told it to flush, and also diff --git a/db/repair.cc b/db/repair.cc index 9ba49fb2f..ad5839e85 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -141,7 +141,7 @@ class Repairer { cf_name + ", id=" + ToString(cf_id)); } Options opts(db_options_, *cf_opts); - MutableCFOptions mut_cf_opts(opts, ImmutableCFOptions(opts)); + MutableCFOptions mut_cf_opts(opts); VersionEdit edit; edit.SetComparatorName(opts.comparator->Name()); diff --git a/db/version_builder_test.cc b/db/version_builder_test.cc index 2a87dc238..501e42d4e 100644 --- a/db/version_builder_test.cc +++ b/db/version_builder_test.cc @@ -29,9 +29,9 @@ class VersionBuilderTest : public testing::Test { : ucmp_(BytewiseComparator()), icmp_(ucmp_), ioptions_(options_), - mutable_cf_options_(options_, ioptions_), + mutable_cf_options_(options_), vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, - nullptr), + nullptr), file_num_(1) { mutable_cf_options_.RefreshDerivedOptions(ioptions_); size_being_compacted_.resize(options_.num_levels); @@ -77,7 +77,7 @@ class VersionBuilderTest : public testing::Test { } void UpdateVersionStorageInfo() { - vstorage_.UpdateFilesByCompactionPri(mutable_cf_options_); + vstorage_.UpdateFilesByCompactionPri(ioptions_.compaction_pri); vstorage_.UpdateNumNonEmptyLevels(); vstorage_.GenerateFileIndexer(); vstorage_.GenerateLevelFilesBrief(); diff --git a/db/version_set.cc b/db/version_set.cc index 8bd6890ac..6d5d4d5df 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1014,7 +1014,7 @@ void Version::PrepareApply( UpdateAccumulatedStats(update_stats); storage_info_.UpdateNumNonEmptyLevels(); storage_info_.CalculateBaseBytes(*cfd_->ioptions(), mutable_cf_options); - storage_info_.UpdateFilesByCompactionPri(mutable_cf_options); + storage_info_.UpdateFilesByCompactionPri(cfd_->ioptions()->compaction_pri); storage_info_.GenerateFileIndexer(); storage_info_.GenerateLevelFilesBrief(); storage_info_.GenerateLevel0NonOverlapping(); @@ -1240,6 +1240,7 @@ void VersionStorageInfo::EstimateCompactionBytesNeeded( } void VersionStorageInfo::ComputeCompactionScore( + const ImmutableCFOptions& immutable_cf_options, const MutableCFOptions& mutable_cf_options) { for (int level = 0; level <= MaxInputLevel(); level++) { double score; @@ -1275,8 +1276,9 @@ void VersionStorageInfo::ComputeCompactionScore( } if (compaction_style_ == kCompactionStyleFIFO) { - score = static_cast(total_size) / - mutable_cf_options.compaction_options_fifo.max_table_files_size; + score = + static_cast(total_size) / + immutable_cf_options.compaction_options_fifo.max_table_files_size; } else { score = static_cast(num_sorted_runs) / mutable_cf_options.level0_file_num_compaction_trigger; @@ -1476,7 +1478,7 @@ void SortFileByOverlappingRatio( } // namespace void VersionStorageInfo::UpdateFilesByCompactionPri( - const MutableCFOptions& mutable_cf_options) { + CompactionPri compaction_pri) { if (compaction_style_ == kCompactionStyleFIFO || compaction_style_ == kCompactionStyleUniversal) { // don't need this @@ -1500,7 +1502,7 @@ void VersionStorageInfo::UpdateFilesByCompactionPri( if (num > temp.size()) { num = temp.size(); } - switch (mutable_cf_options.compaction_pri) { + switch (compaction_pri) { case kByCompensatedSize: std::partial_sort(temp.begin(), temp.begin() + num, temp.end(), CompareCompensatedSizeDescending); @@ -2132,6 +2134,7 @@ void VersionSet::AppendVersion(ColumnFamilyData* column_family_data, Version* v) { // compute new compaction score v->storage_info()->ComputeCompactionScore( + *column_family_data->ioptions(), *column_family_data->GetLatestMutableCFOptions()); // Mark v finalized @@ -2909,7 +2912,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname, vstorage->files_ = new_files_list; vstorage->num_levels_ = new_levels; - MutableCFOptions mutable_cf_options(*options, ImmutableCFOptions(*options)); + MutableCFOptions mutable_cf_options(*options); VersionEdit ve; InstrumentedMutex dummy_mutex; InstrumentedMutexLock l(&dummy_mutex); diff --git a/db/version_set.h b/db/version_set.h index 8fe7c47ce..41f0c4e42 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -120,7 +120,8 @@ class VersionStorageInfo { // We use compaction scores to figure out which compaction to do next // REQUIRES: db_mutex held!! // TODO find a better way to pass compaction_options_fifo. - void ComputeCompactionScore(const MutableCFOptions& mutable_cf_options); + void ComputeCompactionScore(const ImmutableCFOptions& immutable_cf_options, + const MutableCFOptions& mutable_cf_options); // Estimate est_comp_needed_bytes_ void EstimateCompactionBytesNeeded( @@ -135,7 +136,7 @@ class VersionStorageInfo { // Sort all files for this version based on their file size and // record results in files_by_compaction_pri_. The largest files are listed // first. - void UpdateFilesByCompactionPri(const MutableCFOptions& mutable_cf_options); + void UpdateFilesByCompactionPri(CompactionPri compaction_pri); void GenerateLevel0NonOverlapping(); bool level0_non_overlapping() const { diff --git a/db/version_set_test.cc b/db/version_set_test.cc index 98b20a110..3a1764fba 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -109,7 +109,7 @@ class VersionStorageInfoTest : public testing::Test { logger_(new CountingLogger()), options_(GetOptionsWithNumLevels(6, logger_)), ioptions_(options_), - mutable_cf_options_(options_, ioptions_), + mutable_cf_options_(options_), vstorage_(&icmp_, ucmp_, 6, kCompactionStyleLevel, nullptr) {} ~VersionStorageInfoTest() { diff --git a/db/write_batch_test.cc b/db/write_batch_test.cc index 7f9f57b94..275b5671e 100644 --- a/db/write_batch_test.cc +++ b/db/write_batch_test.cc @@ -31,9 +31,8 @@ static std::string PrintContents(WriteBatch* b) { options.memtable_factory = factory; ImmutableCFOptions ioptions(options); WriteBufferManager wb(options.db_write_buffer_size); - MemTable* mem = - new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb, - kMaxSequenceNumber); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber); mem->Ref(); std::string state; ColumnFamilyMemTablesDefault cf_mems_default(mem); diff --git a/java/rocksjni/write_batch_test.cc b/java/rocksjni/write_batch_test.cc index 40b82d9d1..604b50346 100644 --- a/java/rocksjni/write_batch_test.cc +++ b/java/rocksjni/write_batch_test.cc @@ -46,8 +46,7 @@ jbyteArray Java_org_rocksdb_WriteBatchTest_getContents( options.memtable_factory = factory; rocksdb::MemTable* mem = new rocksdb::MemTable( cmp, rocksdb::ImmutableCFOptions(options), - rocksdb::MutableCFOptions(options, rocksdb::ImmutableCFOptions(options)), - &wb, rocksdb::kMaxSequenceNumber); + rocksdb::MutableCFOptions(options), &wb, rocksdb::kMaxSequenceNumber); mem->Ref(); std::string state; rocksdb::ColumnFamilyMemTablesDefault cf_mems_default(mem); diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index ce7772bfd..9e073806f 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -75,7 +75,7 @@ struct SstFileWriter::Rep { const Comparator* _user_comparator) : env_options(_env_options), ioptions(options), - mutable_cf_options(options, ioptions), + mutable_cf_options(options), internal_comparator(_user_comparator) {} std::unique_ptr file_writer; diff --git a/table/table_test.cc b/table/table_test.cc index de885e2a8..572394bb5 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -415,9 +415,9 @@ class MemTableConstructor: public Constructor { table_factory_(new SkipListFactory) { options_.memtable_factory = table_factory_; ImmutableCFOptions ioptions(options_); - memtable_ = new MemTable(internal_comparator_, ioptions, - MutableCFOptions(options_, ioptions), wb, - kMaxSequenceNumber); + memtable_ = + new MemTable(internal_comparator_, ioptions, MutableCFOptions(options_), + wb, kMaxSequenceNumber); memtable_->Ref(); } ~MemTableConstructor() { @@ -430,8 +430,8 @@ class MemTableConstructor: public Constructor { delete memtable_->Unref(); ImmutableCFOptions mem_ioptions(ioptions); memtable_ = new MemTable(internal_comparator_, mem_ioptions, - MutableCFOptions(options_, mem_ioptions), - write_buffer_manager_, kMaxSequenceNumber); + MutableCFOptions(options_), write_buffer_manager_, + kMaxSequenceNumber); memtable_->Ref(); int seq = 1; for (const auto kv : kv_map) { @@ -2410,9 +2410,8 @@ TEST_F(MemTableTest, Simple) { options.memtable_factory = table_factory; ImmutableCFOptions ioptions(options); WriteBufferManager wb(options.db_write_buffer_size); - MemTable* memtable = - new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb, - kMaxSequenceNumber); + MemTable* memtable = new MemTable(cmp, ioptions, MutableCFOptions(options), + &wb, kMaxSequenceNumber); memtable->Ref(); WriteBatch batch; WriteBatchInternal::SetSequence(&batch, 100); diff --git a/util/cf_options.cc b/util/cf_options.cc index a24bc24e8..ebe65be17 100644 --- a/util/cf_options.cc +++ b/util/cf_options.cc @@ -10,8 +10,8 @@ #endif #include -#include #include +#include #include #include "port/port.h" #include "rocksdb/env.h" @@ -21,6 +21,7 @@ namespace rocksdb { ImmutableCFOptions::ImmutableCFOptions(const Options& options) : compaction_style(options.compaction_style), + compaction_pri(options.compaction_pri), compaction_options_universal(options.compaction_options_universal), compaction_options_fifo(options.compaction_options_fifo), prefix_extractor(options.prefix_extractor.get()), @@ -58,7 +59,8 @@ ImmutableCFOptions::ImmutableCFOptions(const Options& options) num_levels(options.num_levels), optimize_filters_for_hits(options.optimize_filters_for_hits), listeners(options.listeners), - row_cache(options.row_cache) {} + row_cache(options.row_cache), + max_subcompactions(options.max_subcompactions) {} // Multiple two operands. If they overflow, return op1. uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) { @@ -75,11 +77,11 @@ uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) { return op1 * casted_op2; } -void MutableCFOptions::RefreshDerivedOptions( - const ImmutableCFOptions& ioptions) { - max_file_size.resize(ioptions.num_levels); - for (int i = 0; i < ioptions.num_levels; ++i) { - if (i == 0 && ioptions.compaction_style == kCompactionStyleUniversal) { +void MutableCFOptions::RefreshDerivedOptions(int num_levels, + CompactionStyle compaction_style) { + max_file_size.resize(num_levels); + for (int i = 0; i < num_levels; ++i) { + if (i == 0 && compaction_style == kCompactionStyleUniversal) { max_file_size[i] = ULLONG_MAX; } else if (i > 1) { max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1], @@ -106,10 +108,12 @@ void MutableCFOptions::Dump(Logger* log) const { arena_block_size); Log(log, " memtable_prefix_bloom_ratio: %f", memtable_prefix_bloom_size_ratio); - Log(log, " memtable_huge_page_size: %" ROCKSDB_PRIszt, + Log(log, " memtable_huge_page_size: %" ROCKSDB_PRIszt, memtable_huge_page_size); Log(log, " max_successive_merges: %" ROCKSDB_PRIszt, max_successive_merges); + Log(log, " inplace_update_num_locks: %" ROCKSDB_PRIszt, + inplace_update_num_locks); Log(log, " disable_auto_compactions: %d", disable_auto_compactions); Log(log, " soft_pending_compaction_bytes_limit: %" PRIu64, @@ -144,6 +148,13 @@ void MutableCFOptions::Dump(Logger* log) const { verify_checksums_in_compaction); Log(log, " max_sequential_skip_in_iterations: %" PRIu64, max_sequential_skip_in_iterations); + Log(log, " paranoid_file_checks: %d", + paranoid_file_checks); + Log(log, " report_bg_io_stats: %d", report_bg_io_stats); + Log(log, " compression: %d", + static_cast(compression)); + Log(log, " min_partial_merge_operands: %" PRIu32, + min_partial_merge_operands); } } // namespace rocksdb diff --git a/util/cf_options.h b/util/cf_options.h index 6a9db52d6..c08534635 100644 --- a/util/cf_options.h +++ b/util/cf_options.h @@ -22,6 +22,8 @@ struct ImmutableCFOptions { CompactionStyle compaction_style; + CompactionPri compaction_pri; + CompactionOptionsUniversal compaction_options_universal; CompactionOptionsFIFO compaction_options_fifo; @@ -102,10 +104,12 @@ struct ImmutableCFOptions { std::vector> listeners; std::shared_ptr row_cache; + + uint32_t max_subcompactions; }; struct MutableCFOptions { - MutableCFOptions(const Options& options, const ImmutableCFOptions& ioptions) + explicit MutableCFOptions(const ColumnFamilyOptions& options) : write_buffer_size(options.write_buffer_size), max_write_buffer_number(options.max_write_buffer_number), arena_block_size(options.arena_block_size), @@ -123,7 +127,6 @@ struct MutableCFOptions { options.level0_file_num_compaction_trigger), level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), level0_stop_writes_trigger(options.level0_stop_writes_trigger), - compaction_pri(options.compaction_pri), max_compaction_bytes(options.max_compaction_bytes), target_file_size_base(options.target_file_size_base), target_file_size_multiplier(options.target_file_size_multiplier), @@ -132,16 +135,15 @@ struct MutableCFOptions { max_bytes_for_level_multiplier_additional( options.max_bytes_for_level_multiplier_additional), verify_checksums_in_compaction(options.verify_checksums_in_compaction), - max_subcompactions(options.max_subcompactions), max_sequential_skip_in_iterations( options.max_sequential_skip_in_iterations), paranoid_file_checks(options.paranoid_file_checks), report_bg_io_stats(options.report_bg_io_stats), compression(options.compression), - min_partial_merge_operands(options.min_partial_merge_operands), - compaction_options_fifo(ioptions.compaction_options_fifo) { - RefreshDerivedOptions(ioptions); + min_partial_merge_operands(options.min_partial_merge_operands) { + RefreshDerivedOptions(options.num_levels, options.compaction_style); } + MutableCFOptions() : write_buffer_size(0), max_write_buffer_number(0), @@ -156,14 +158,12 @@ struct MutableCFOptions { level0_file_num_compaction_trigger(0), level0_slowdown_writes_trigger(0), level0_stop_writes_trigger(0), - compaction_pri(kByCompensatedSize), max_compaction_bytes(0), target_file_size_base(0), target_file_size_multiplier(0), max_bytes_for_level_base(0), max_bytes_for_level_multiplier(0), verify_checksums_in_compaction(false), - max_subcompactions(1), max_sequential_skip_in_iterations(0), paranoid_file_checks(false), report_bg_io_stats(false), @@ -171,7 +171,11 @@ struct MutableCFOptions { min_partial_merge_operands(2) {} // Must be called after any change to MutableCFOptions - void RefreshDerivedOptions(const ImmutableCFOptions& ioptions); + void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style); + + void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) { + RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style); + } // Get the max file size in a given level. uint64_t MaxFileSizeForLevel(int level) const; @@ -201,7 +205,6 @@ struct MutableCFOptions { int level0_file_num_compaction_trigger; int level0_slowdown_writes_trigger; int level0_stop_writes_trigger; - CompactionPri compaction_pri; uint64_t max_compaction_bytes; uint64_t target_file_size_base; int target_file_size_multiplier; @@ -209,7 +212,6 @@ struct MutableCFOptions { int max_bytes_for_level_multiplier; std::vector max_bytes_for_level_multiplier_additional; bool verify_checksums_in_compaction; - int max_subcompactions; // Misc options uint64_t max_sequential_skip_in_iterations; @@ -217,7 +219,6 @@ struct MutableCFOptions { bool report_bg_io_stats; CompressionType compression; uint32_t min_partial_merge_operands; - CompactionOptionsFIFO compaction_options_fifo; // Derived options // Per-level target file size.