Refactor MutableCFOptions

Summary:
* Change constructor of MutableCFOptions to depends only on ColumnFamilyOptions.
* Move `max_subcompactions`, `compaction_options_fifo` and `compaction_pri` to ImmutableCFOptions to make it clear that they are immutable.

Test Plan: existing unit tests.

Reviewers: yhchiang, IslamAbdelRahman, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D63945

db/column_family.cc

@@ -350,7 +350,7 @@ ColumnFamilyData::ColumnFamilyData(
       options_(*db_options,
                SanitizeOptions(*db_options, &internal_comparator_, cf_options)),
       ioptions_(options_),
-      mutable_cf_options_(options_, ioptions_),
+      mutable_cf_options_(options_),
       write_buffer_manager_(write_buffer_manager),
       mem_(nullptr),
       imm_(options_.min_write_buffer_number_to_merge,

db/column_family_test.cc

@@ -2440,9 +2440,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
 
   VersionStorageInfo* vstorage = cfd->current()->storage_info();
 
-  MutableCFOptions mutable_cf_options(
+  MutableCFOptions mutable_cf_options(column_family_options_);
-      Options(db_options_, column_family_options_),
-      ImmutableCFOptions(Options(db_options_, column_family_options_)));
 
   mutable_cf_options.level0_slowdown_writes_trigger = 20;
   mutable_cf_options.level0_stop_writes_trigger = 10000;
@@ -2631,9 +2629,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   VersionStorageInfo* vstorage = cfd->current()->storage_info();
 
-  MutableCFOptions mutable_cf_options(
+  MutableCFOptions mutable_cf_options(column_family_options_);
-      Options(db_options_, column_family_options_),
-      ImmutableCFOptions(Options(db_options_, column_family_options_)));
 
   // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
   mutable_cf_options.level0_file_num_compaction_trigger = 4;
@@ -2702,9 +2698,7 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
       static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
   VersionStorageInfo* vstorage1 = cfd1->current()->storage_info();
 
-  MutableCFOptions mutable_cf_options(
+  MutableCFOptions mutable_cf_options(column_family_options_);
-      Options(db_options_, column_family_options_),
-      ImmutableCFOptions(Options(db_options_, column_family_options_)));
   mutable_cf_options.level0_slowdown_writes_trigger = 20;
   mutable_cf_options.level0_stop_writes_trigger = 10000;
   mutable_cf_options.soft_pending_compaction_bytes_limit = 200;
@@ -2786,9 +2780,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
       static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
   VersionStorageInfo* vstorage1 = cfd1->current()->storage_info();
 
-  MutableCFOptions mutable_cf_options(
+  MutableCFOptions mutable_cf_options(column_family_options_);
-      Options(db_options_, column_family_options_),
-      ImmutableCFOptions(Options(db_options_, column_family_options_)));
   // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
   mutable_cf_options.level0_file_num_compaction_trigger = 4;
   mutable_cf_options.level0_slowdown_writes_trigger = 36;

db/compaction.cc

@@ -140,6 +140,7 @@ bool Compaction::IsFullCompaction(
 }
 
 Compaction::Compaction(VersionStorageInfo* vstorage,
+                       const ImmutableCFOptions& _immutable_cf_options,
                        const MutableCFOptions& _mutable_cf_options,
                        std::vector<CompactionInputFiles> _inputs,
                        int _output_level, uint64_t _target_file_size,
@@ -153,6 +154,7 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
       output_level_(_output_level),
       max_output_file_size_(_target_file_size),
       max_compaction_bytes_(_max_compaction_bytes),
+      immutable_cf_options_(_immutable_cf_options),
       mutable_cf_options_(_mutable_cf_options),
       input_version_(nullptr),
       number_levels_(vstorage->num_levels()),
@@ -427,7 +429,7 @@ bool Compaction::IsOutputLevelEmpty() const {
 }
 
 bool Compaction::ShouldFormSubcompactions() const {
-  if (mutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
+  if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
     return false;
   }
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {

db/compaction.h

@@ -35,6 +35,7 @@ class CompactionFilter;
 class Compaction {
  public:
   Compaction(VersionStorageInfo* input_version,
+             const ImmutableCFOptions& immutable_cf_options,
              const MutableCFOptions& mutable_cf_options,
              std::vector<CompactionInputFiles> inputs, int output_level,
              uint64_t target_file_size, uint64_t max_compaction_bytes,
@@ -171,6 +172,12 @@ class Compaction {
   // How many total levels are there?
   int number_levels() const { return number_levels_; }
 
+  // Return the ImmutableCFOptions that should be used throughout the compaction
+  // procedure
+  const ImmutableCFOptions* immutable_cf_options() const {
+    return &immutable_cf_options_;
+  }
+
   // Return the MutableCFOptions that should be used throughout the compaction
   // procedure
   const MutableCFOptions* mutable_cf_options() const {
@@ -253,7 +260,8 @@ class Compaction {
   const int output_level_;  // levels to which output files are stored
   uint64_t max_output_file_size_;
   uint64_t max_compaction_bytes_;
-  MutableCFOptions mutable_cf_options_;
+  const ImmutableCFOptions immutable_cf_options_;
+  const MutableCFOptions mutable_cf_options_;
   Version* input_version_;
   VersionEdit edit_;
   const int number_levels_;

db/compaction_job_test.cc

@@ -68,7 +68,7 @@ class CompactionJobTest : public testing::Test {
   CompactionJobTest()
       : env_(Env::Default()),
         dbname_(test::TmpDir() + "/compaction_job_test"),
-        mutable_cf_options_(Options(), ImmutableCFOptions(Options())),
+        mutable_cf_options_(Options()),
         table_cache_(NewLRUCache(50000, 16)),
         write_buffer_manager_(db_options_.db_write_buffer_size),
         versions_(new VersionSet(dbname_, &db_options_, env_options_,
@@ -239,7 +239,7 @@ class CompactionJobTest : public testing::Test {
       num_input_files += level_files.size();
     }
 
-    Compaction compaction(cfd->current()->storage_info(),
+    Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(),
                           *cfd->GetLatestMutableCFOptions(),
                           compaction_input_files, 1, 1024 * 1024,
                           10 * 1024 * 1024, 0, kNoCompression, {}, true);

db/compaction_picker.cc

@@ -275,8 +275,8 @@ Compaction* CompactionPicker::FormCompaction(
     return nullptr;
   }
   auto c =
-      new Compaction(vstorage, mutable_cf_options, input_files, output_level,
+      new Compaction(vstorage, ioptions_, mutable_cf_options, input_files,
-                     compact_options.output_file_size_limit,
+                     output_level, compact_options.output_file_size_limit,
                      mutable_cf_options.max_compaction_bytes, output_path_id,
                      compact_options.compression, /* grandparents */ {}, true);
 
@@ -507,8 +507,8 @@ Compaction* CompactionPicker::CompactRange(
       }
     }
     Compaction* c = new Compaction(
-        vstorage, mutable_cf_options, std::move(inputs), output_level,
+        vstorage, ioptions_, mutable_cf_options, std::move(inputs),
-        mutable_cf_options.MaxFileSizeForLevel(output_level),
+        output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
         /* max_compaction_bytes */ LLONG_MAX, output_path_id,
         GetCompressionType(ioptions_, vstorage, mutable_cf_options,
                            output_level, 1),
@@ -607,8 +607,8 @@ Compaction* CompactionPicker::CompactRange(
   std::vector<FileMetaData*> grandparents;
   GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
   Compaction* compaction = new Compaction(
-      vstorage, mutable_cf_options, std::move(compaction_inputs), output_level,
+      vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
-      mutable_cf_options.MaxFileSizeForLevel(output_level),
+      output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
       mutable_cf_options.max_compaction_bytes, output_path_id,
       GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
                          vstorage->base_level()),
@@ -623,7 +623,7 @@ Compaction* CompactionPicker::CompactRange(
   // takes running compactions into account (by skipping files that are already
   // being compacted). Since we just changed compaction score, we recalculate it
   // here
-  vstorage->ComputeCompactionScore(mutable_cf_options);
+  vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
 
   return compaction;
 }
@@ -1012,8 +1012,8 @@ Compaction* LevelCompactionPicker::PickCompaction(
   std::vector<FileMetaData*> grandparents;
   GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
   auto c = new Compaction(
-      vstorage, mutable_cf_options, std::move(compaction_inputs), output_level,
+      vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
-      mutable_cf_options.MaxFileSizeForLevel(output_level),
+      output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
       mutable_cf_options.max_compaction_bytes,
       GetPathId(ioptions_, mutable_cf_options, output_level),
       GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
@@ -1031,7 +1031,7 @@ Compaction* LevelCompactionPicker::PickCompaction(
   // takes running compactions into account (by skipping files that are already
   // being compacted). Since we just changed compaction score, we recalculate it
   // here
-  vstorage->ComputeCompactionScore(mutable_cf_options);
+  vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
 
   TEST_SYNC_POINT_CALLBACK("LevelCompactionPicker::PickCompaction:Return", c);
 
@@ -1643,7 +1643,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
     compaction_reason = CompactionReason::kUniversalSizeRatio;
   }
   return new Compaction(
-      vstorage, mutable_cf_options, std::move(inputs), output_level,
+      vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
       mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
       GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
                          1, enable_compression),
@@ -1767,7 +1767,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
   }
 
   return new Compaction(
-      vstorage, mutable_cf_options, std::move(inputs),
+      vstorage, ioptions_, mutable_cf_options, std::move(inputs),
       vstorage->num_levels() - 1,
       mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1),
       /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
@@ -1832,7 +1832,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
     }
   }
   Compaction* c = new Compaction(
-      vstorage, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
+      vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
       kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
       /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
   level0_compactions_in_progress_.insert(c);

db/compaction_picker_test.cc

@@ -48,7 +48,7 @@ class CompactionPickerTest : public testing::Test {
       : ucmp_(BytewiseComparator()),
         icmp_(ucmp_),
         ioptions_(options_),
-        mutable_cf_options_(options_, ioptions_),
+        mutable_cf_options_(options_),
         level_compaction_picker(ioptions_, &icmp_),
         cf_name_("dummy"),
         log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
@@ -115,11 +115,11 @@ class CompactionPickerTest : public testing::Test {
 
   void UpdateVersionStorageInfo() {
     vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
-    vstorage_->UpdateFilesByCompactionPri(mutable_cf_options_);
+    vstorage_->UpdateFilesByCompactionPri(ioptions_.compaction_pri);
     vstorage_->UpdateNumNonEmptyLevels();
     vstorage_->GenerateFileIndexer();
     vstorage_->GenerateLevelFilesBrief();
-    vstorage_->ComputeCompactionScore(mutable_cf_options_);
+    vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_);
     vstorage_->GenerateLevel0NonOverlapping();
     vstorage_->SetFinalized();
   }
@@ -350,10 +350,10 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic4) {
 TEST_F(CompactionPickerTest, LevelTriggerDynamic4) {
   int num_levels = ioptions_.num_levels;
   ioptions_.level_compaction_dynamic_level_bytes = true;
+  ioptions_.compaction_pri = kMinOverlappingRatio;
   mutable_cf_options_.level0_file_num_compaction_trigger = 2;
   mutable_cf_options_.max_bytes_for_level_base = 200;
   mutable_cf_options_.max_bytes_for_level_multiplier = 10;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
   NewVersionStorage(num_levels, kCompactionStyleLevel);
   Add(0, 1U, "150", "200");
   Add(num_levels - 1, 3U, "200", "250", 300U);
@@ -489,10 +489,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
 
 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
   NewVersionStorage(6, kCompactionStyleLevel);
+  ioptions_.compaction_pri = kMinOverlappingRatio;
   mutable_cf_options_.target_file_size_base = 10000000;
   mutable_cf_options_.target_file_size_multiplier = 10;
   mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
 
   Add(2, 6U, "150", "179", 50000000U);
   Add(2, 7U, "180", "220", 50000000U);
@@ -516,10 +516,10 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
 
 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
   NewVersionStorage(6, kCompactionStyleLevel);
+  ioptions_.compaction_pri = kMinOverlappingRatio;
   mutable_cf_options_.target_file_size_base = 10000000;
   mutable_cf_options_.target_file_size_multiplier = 10;
   mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
 
   Add(2, 6U, "150", "175",
       60000000U);  // Overlaps with file 26, 27, total size 521M
@@ -546,9 +546,9 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
 
 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
   NewVersionStorage(6, kCompactionStyleLevel);
+  ioptions_.compaction_pri = kMinOverlappingRatio;
   mutable_cf_options_.max_bytes_for_level_base = 10000000;
   mutable_cf_options_.max_bytes_for_level_multiplier = 10;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
 
   // file 7 and 8 over lap with the same file, but file 8 is smaller so
   // it will be picked.
@@ -598,7 +598,7 @@ TEST_F(CompactionPickerTest, ParentIndexResetBug) {
 // ranges (with different sequence numbers) in the input files.
 TEST_F(CompactionPickerTest, OverlappingUserKeys) {
   NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.compaction_pri = kByCompensatedSize;
+  ioptions_.compaction_pri = kByCompensatedSize;
 
   Add(1, 1U, "100", "150", 1U);
   // Overlapping user keys

db/db_impl.cc

@@ -2263,7 +2263,8 @@ Status DBImpl::CompactFilesImpl(
   // takes running compactions into account (by skipping files that are already
   // being compacted). Since we just changed compaction score, we recalculate it
   // here.
-  version->storage_info()->ComputeCompactionScore(*c->mutable_cf_options());
+  version->storage_info()->ComputeCompactionScore(*cfd->ioptions(),
+                                                  *c->mutable_cf_options());
 
   compaction_job.Prepare();
 
@@ -4037,9 +4038,8 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
       write_thread_.EnterUnbatched(&w, &mutex_);
       // LogAndApply will both write the creation in MANIFEST and create
       // ColumnFamilyData object
-      s = versions_->LogAndApply(
+      s = versions_->LogAndApply(nullptr, MutableCFOptions(opt), &edit, &mutex_,
-          nullptr, MutableCFOptions(opt, ImmutableCFOptions(opt)), &edit,
+                                 directories_.GetDbDir(), false, &cf_options);
-          &mutex_, directories_.GetDbDir(), false, &cf_options);
 
       if (s.ok()) {
         // If the column family was created successfully, we then persist

db/db_impl_experimental.cc

@@ -49,7 +49,8 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
     }
     // Since we have some more files to compact, we should also recompute
     // compaction score
-    vstorage->ComputeCompactionScore(*cfd->GetLatestMutableCFOptions());
+    vstorage->ComputeCompactionScore(*cfd->ioptions(),
+                                     *cfd->GetLatestMutableCFOptions());
     SchedulePendingCompaction(cfd);
     MaybeScheduleFlushOrCompaction();
   }

db/memtable_list_test.cc

@@ -127,9 +127,8 @@ TEST_F(MemTableListTest, GetTest) {
   ImmutableCFOptions ioptions(options);
 
   WriteBufferManager wb(options.db_write_buffer_size);
-  MemTable* mem =
+  MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-      new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
+                               kMaxSequenceNumber);
-                   kMaxSequenceNumber);
   mem->Ref();
 
   // Write some keys to this memtable.
@@ -164,9 +163,8 @@ TEST_F(MemTableListTest, GetTest) {
 
   // Create another memtable and write some keys to it
   WriteBufferManager wb2(options.db_write_buffer_size);
-  MemTable* mem2 =
+  MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2,
-      new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb2,
+                                kMaxSequenceNumber);
-                   kMaxSequenceNumber);
   mem2->Ref();
 
   mem2->Add(++seq, kTypeDeletion, "key1", "");
@@ -229,9 +227,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
   ImmutableCFOptions ioptions(options);
 
   WriteBufferManager wb(options.db_write_buffer_size);
-  MemTable* mem =
+  MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-      new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
+                               kMaxSequenceNumber);
-                   kMaxSequenceNumber);
   mem->Ref();
 
   // Write some keys to this memtable.
@@ -272,8 +269,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
   list.PickMemtablesToFlush(&to_flush);
   ASSERT_EQ(1, to_flush.size());
 
-  s = Mock_InstallMemtableFlushResults(
+  s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options),
-      &list, MutableCFOptions(options, ioptions), to_flush, &to_delete);
+                                       to_flush, &to_delete);
   ASSERT_OK(s);
   ASSERT_EQ(0, list.NumNotFlushed());
   ASSERT_EQ(1, list.NumFlushed());
@@ -304,9 +301,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
 
   // Create another memtable and write some keys to it
   WriteBufferManager wb2(options.db_write_buffer_size);
-  MemTable* mem2 =
+  MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2,
-      new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb2,
+                                kMaxSequenceNumber);
-                   kMaxSequenceNumber);
   mem2->Ref();
 
   mem2->Add(++seq, kTypeDeletion, "key1", "");
@@ -321,8 +317,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
   ASSERT_EQ(1, to_flush.size());
 
   // Flush second memtable
-  s = Mock_InstallMemtableFlushResults(
+  s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options),
-      &list, MutableCFOptions(options, ioptions), to_flush, &to_delete);
+                                       to_flush, &to_delete);
   ASSERT_OK(s);
   ASSERT_EQ(0, list.NumNotFlushed());
   ASSERT_EQ(2, list.NumFlushed());
@@ -330,9 +326,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
 
   // Add a third memtable to push the first memtable out of the history
   WriteBufferManager wb3(options.db_write_buffer_size);
-  MemTable* mem3 =
+  MemTable* mem3 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb3,
-      new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb3,
+                                kMaxSequenceNumber);
-                   kMaxSequenceNumber);
   mem3->Ref();
   list.Add(mem3, &to_delete);
   ASSERT_EQ(1, list.NumNotFlushed());
@@ -401,7 +396,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
 
   // Create some MemTables
   std::vector<MemTable*> tables;
-  MutableCFOptions mutable_cf_options(options, ioptions);
+  MutableCFOptions mutable_cf_options(options);
   for (int i = 0; i < num_tables; i++) {
     MemTable* mem = new MemTable(cmp, ioptions, mutable_cf_options, &wb,
                                  kMaxSequenceNumber);
@@ -539,8 +534,8 @@ TEST_F(MemTableListTest, FlushPendingTest) {
   ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
 
   // Flush the 4 memtables that were picked in to_flush
-  s = Mock_InstallMemtableFlushResults(
+  s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options),
-      &list, MutableCFOptions(options, ioptions), to_flush, &to_delete);
+                                       to_flush, &to_delete);
   ASSERT_OK(s);
 
   // Note:  now to_flush contains tables[0,1,2,4].  to_flush2 contains
@@ -560,7 +555,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
 
   // Flush the 1 memtable that was picked in to_flush2
   s = MemTableListTest::Mock_InstallMemtableFlushResults(
-      &list, MutableCFOptions(options, ioptions), to_flush2, &to_delete);
+      &list, MutableCFOptions(options), to_flush2, &to_delete);
   ASSERT_OK(s);
 
   // This will actually install 2 tables.  The 1 we told it to flush, and also

db/repair.cc

@@ -141,7 +141,7 @@ class Repairer {
                                 cf_name + ", id=" + ToString(cf_id));
     }
     Options opts(db_options_, *cf_opts);
-    MutableCFOptions mut_cf_opts(opts, ImmutableCFOptions(opts));
+    MutableCFOptions mut_cf_opts(opts);
 
     VersionEdit edit;
     edit.SetComparatorName(opts.comparator->Name());

db/version_builder_test.cc

@@ -29,9 +29,9 @@ class VersionBuilderTest : public testing::Test {
       : ucmp_(BytewiseComparator()),
         icmp_(ucmp_),
         ioptions_(options_),
-        mutable_cf_options_(options_, ioptions_),
+        mutable_cf_options_(options_),
         vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
-                 nullptr),
+                  nullptr),
         file_num_(1) {
     mutable_cf_options_.RefreshDerivedOptions(ioptions_);
     size_being_compacted_.resize(options_.num_levels);
@@ -77,7 +77,7 @@ class VersionBuilderTest : public testing::Test {
   }
 
   void UpdateVersionStorageInfo() {
-    vstorage_.UpdateFilesByCompactionPri(mutable_cf_options_);
+    vstorage_.UpdateFilesByCompactionPri(ioptions_.compaction_pri);
     vstorage_.UpdateNumNonEmptyLevels();
     vstorage_.GenerateFileIndexer();
     vstorage_.GenerateLevelFilesBrief();

db/version_set.cc

@@ -1014,7 +1014,7 @@ void Version::PrepareApply(
   UpdateAccumulatedStats(update_stats);
   storage_info_.UpdateNumNonEmptyLevels();
   storage_info_.CalculateBaseBytes(*cfd_->ioptions(), mutable_cf_options);
-  storage_info_.UpdateFilesByCompactionPri(mutable_cf_options);
+  storage_info_.UpdateFilesByCompactionPri(cfd_->ioptions()->compaction_pri);
   storage_info_.GenerateFileIndexer();
   storage_info_.GenerateLevelFilesBrief();
   storage_info_.GenerateLevel0NonOverlapping();
@@ -1240,6 +1240,7 @@ void VersionStorageInfo::EstimateCompactionBytesNeeded(
 }
 
 void VersionStorageInfo::ComputeCompactionScore(
+    const ImmutableCFOptions& immutable_cf_options,
     const MutableCFOptions& mutable_cf_options) {
   for (int level = 0; level <= MaxInputLevel(); level++) {
     double score;
@@ -1275,8 +1276,9 @@ void VersionStorageInfo::ComputeCompactionScore(
       }
 
       if (compaction_style_ == kCompactionStyleFIFO) {
-        score = static_cast<double>(total_size) /
+        score =
-                mutable_cf_options.compaction_options_fifo.max_table_files_size;
+            static_cast<double>(total_size) /
+            immutable_cf_options.compaction_options_fifo.max_table_files_size;
       } else {
         score = static_cast<double>(num_sorted_runs) /
                 mutable_cf_options.level0_file_num_compaction_trigger;
@@ -1476,7 +1478,7 @@ void SortFileByOverlappingRatio(
 }  // namespace
 
 void VersionStorageInfo::UpdateFilesByCompactionPri(
-    const MutableCFOptions& mutable_cf_options) {
+    CompactionPri compaction_pri) {
   if (compaction_style_ == kCompactionStyleFIFO ||
       compaction_style_ == kCompactionStyleUniversal) {
     // don't need this
@@ -1500,7 +1502,7 @@ void VersionStorageInfo::UpdateFilesByCompactionPri(
     if (num > temp.size()) {
       num = temp.size();
     }
-    switch (mutable_cf_options.compaction_pri) {
+    switch (compaction_pri) {
       case kByCompensatedSize:
         std::partial_sort(temp.begin(), temp.begin() + num, temp.end(),
                           CompareCompensatedSizeDescending);
@@ -2132,6 +2134,7 @@ void VersionSet::AppendVersion(ColumnFamilyData* column_family_data,
                                Version* v) {
   // compute new compaction score
   v->storage_info()->ComputeCompactionScore(
+      *column_family_data->ioptions(),
       *column_family_data->GetLatestMutableCFOptions());
 
   // Mark v finalized
@@ -2909,7 +2912,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
   vstorage->files_ = new_files_list;
   vstorage->num_levels_ = new_levels;
 
-  MutableCFOptions mutable_cf_options(*options, ImmutableCFOptions(*options));
+  MutableCFOptions mutable_cf_options(*options);
   VersionEdit ve;
   InstrumentedMutex dummy_mutex;
   InstrumentedMutexLock l(&dummy_mutex);

db/version_set.h

@@ -120,7 +120,8 @@ class VersionStorageInfo {
   // We use compaction scores to figure out which compaction to do next
   // REQUIRES: db_mutex held!!
   // TODO find a better way to pass compaction_options_fifo.
-  void ComputeCompactionScore(const MutableCFOptions& mutable_cf_options);
+  void ComputeCompactionScore(const ImmutableCFOptions& immutable_cf_options,
+                              const MutableCFOptions& mutable_cf_options);
 
   // Estimate est_comp_needed_bytes_
   void EstimateCompactionBytesNeeded(
@@ -135,7 +136,7 @@ class VersionStorageInfo {
   // Sort all files for this version based on their file size and
   // record results in files_by_compaction_pri_. The largest files are listed
   // first.
-  void UpdateFilesByCompactionPri(const MutableCFOptions& mutable_cf_options);
+  void UpdateFilesByCompactionPri(CompactionPri compaction_pri);
 
   void GenerateLevel0NonOverlapping();
   bool level0_non_overlapping() const {

db/version_set_test.cc

@@ -109,7 +109,7 @@ class VersionStorageInfoTest : public testing::Test {
         logger_(new CountingLogger()),
         options_(GetOptionsWithNumLevels(6, logger_)),
         ioptions_(options_),
-        mutable_cf_options_(options_, ioptions_),
+        mutable_cf_options_(options_),
         vstorage_(&icmp_, ucmp_, 6, kCompactionStyleLevel, nullptr) {}
 
   ~VersionStorageInfoTest() {

db/write_batch_test.cc

@@ -31,9 +31,8 @@ static std::string PrintContents(WriteBatch* b) {
   options.memtable_factory = factory;
   ImmutableCFOptions ioptions(options);
   WriteBufferManager wb(options.db_write_buffer_size);
-  MemTable* mem =
+  MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-      new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
+                               kMaxSequenceNumber);
-                   kMaxSequenceNumber);
   mem->Ref();
   std::string state;
   ColumnFamilyMemTablesDefault cf_mems_default(mem);

java/rocksjni/write_batch_test.cc

@@ -46,8 +46,7 @@ jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(
   options.memtable_factory = factory;
   rocksdb::MemTable* mem = new rocksdb::MemTable(
       cmp, rocksdb::ImmutableCFOptions(options),
-      rocksdb::MutableCFOptions(options, rocksdb::ImmutableCFOptions(options)),
+      rocksdb::MutableCFOptions(options), &wb, rocksdb::kMaxSequenceNumber);
-      &wb, rocksdb::kMaxSequenceNumber);
   mem->Ref();
   std::string state;
   rocksdb::ColumnFamilyMemTablesDefault cf_mems_default(mem);

table/sst_file_writer.cc

@@ -75,7 +75,7 @@ struct SstFileWriter::Rep {
       const Comparator* _user_comparator)
       : env_options(_env_options),
         ioptions(options),
-        mutable_cf_options(options, ioptions),
+        mutable_cf_options(options),
         internal_comparator(_user_comparator) {}
 
   std::unique_ptr<WritableFileWriter> file_writer;

table/table_test.cc

@@ -415,9 +415,9 @@ class MemTableConstructor: public Constructor {
         table_factory_(new SkipListFactory) {
     options_.memtable_factory = table_factory_;
     ImmutableCFOptions ioptions(options_);
-    memtable_ = new MemTable(internal_comparator_, ioptions,
+    memtable_ =
-                             MutableCFOptions(options_, ioptions), wb,
+        new MemTable(internal_comparator_, ioptions, MutableCFOptions(options_),
-                             kMaxSequenceNumber);
+                     wb, kMaxSequenceNumber);
     memtable_->Ref();
   }
   ~MemTableConstructor() {
@@ -430,8 +430,8 @@ class MemTableConstructor: public Constructor {
     delete memtable_->Unref();
     ImmutableCFOptions mem_ioptions(ioptions);
     memtable_ = new MemTable(internal_comparator_, mem_ioptions,
-                             MutableCFOptions(options_, mem_ioptions),
+                             MutableCFOptions(options_), write_buffer_manager_,
-                             write_buffer_manager_, kMaxSequenceNumber);
+                             kMaxSequenceNumber);
     memtable_->Ref();
     int seq = 1;
     for (const auto kv : kv_map) {
@@ -2410,9 +2410,8 @@ TEST_F(MemTableTest, Simple) {
   options.memtable_factory = table_factory;
   ImmutableCFOptions ioptions(options);
   WriteBufferManager wb(options.db_write_buffer_size);
-  MemTable* memtable =
+  MemTable* memtable = new MemTable(cmp, ioptions, MutableCFOptions(options),
-      new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
+                                    &wb, kMaxSequenceNumber);
-                   kMaxSequenceNumber);
   memtable->Ref();
   WriteBatch batch;
   WriteBatchInternal::SetSequence(&batch, 100);

util/cf_options.cc

@@ -10,8 +10,8 @@
 #endif
 
 #include <inttypes.h>
-#include <limits>
 #include <cassert>
+#include <limits>
 #include <string>
 #include "port/port.h"
 #include "rocksdb/env.h"
@@ -21,6 +21,7 @@ namespace rocksdb {
 
 ImmutableCFOptions::ImmutableCFOptions(const Options& options)
     : compaction_style(options.compaction_style),
+      compaction_pri(options.compaction_pri),
       compaction_options_universal(options.compaction_options_universal),
       compaction_options_fifo(options.compaction_options_fifo),
       prefix_extractor(options.prefix_extractor.get()),
@@ -58,7 +59,8 @@ ImmutableCFOptions::ImmutableCFOptions(const Options& options)
       num_levels(options.num_levels),
       optimize_filters_for_hits(options.optimize_filters_for_hits),
       listeners(options.listeners),
-      row_cache(options.row_cache) {}
+      row_cache(options.row_cache),
+      max_subcompactions(options.max_subcompactions) {}
 
 // Multiple two operands. If they overflow, return op1.
 uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
@@ -75,11 +77,11 @@ uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
   return op1 * casted_op2;
 }
 
-void MutableCFOptions::RefreshDerivedOptions(
+void MutableCFOptions::RefreshDerivedOptions(int num_levels,
-    const ImmutableCFOptions& ioptions) {
+                                             CompactionStyle compaction_style) {
-  max_file_size.resize(ioptions.num_levels);
+  max_file_size.resize(num_levels);
-  for (int i = 0; i < ioptions.num_levels; ++i) {
+  for (int i = 0; i < num_levels; ++i) {
-    if (i == 0 && ioptions.compaction_style == kCompactionStyleUniversal) {
+    if (i == 0 && compaction_style == kCompactionStyleUniversal) {
       max_file_size[i] = ULLONG_MAX;
     } else if (i > 1) {
       max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1],
@@ -106,10 +108,12 @@ void MutableCFOptions::Dump(Logger* log) const {
       arena_block_size);
   Log(log, "              memtable_prefix_bloom_ratio: %f",
       memtable_prefix_bloom_size_ratio);
-  Log(log, " memtable_huge_page_size: %" ROCKSDB_PRIszt,
+  Log(log, "                  memtable_huge_page_size: %" ROCKSDB_PRIszt,
       memtable_huge_page_size);
   Log(log, "                    max_successive_merges: %" ROCKSDB_PRIszt,
       max_successive_merges);
+  Log(log, "                 inplace_update_num_locks: %" ROCKSDB_PRIszt,
+      inplace_update_num_locks);
   Log(log, "                 disable_auto_compactions: %d",
       disable_auto_compactions);
   Log(log, "      soft_pending_compaction_bytes_limit: %" PRIu64,
@@ -144,6 +148,13 @@ void MutableCFOptions::Dump(Logger* log) const {
       verify_checksums_in_compaction);
   Log(log, "        max_sequential_skip_in_iterations: %" PRIu64,
       max_sequential_skip_in_iterations);
+  Log(log, "                     paranoid_file_checks: %d",
+      paranoid_file_checks);
+  Log(log, "                       report_bg_io_stats: %d", report_bg_io_stats);
+  Log(log, "                              compression: %d",
+      static_cast<int>(compression));
+  Log(log, "               min_partial_merge_operands: %" PRIu32,
+      min_partial_merge_operands);
 }
 
 }  // namespace rocksdb

util/cf_options.h

@@ -22,6 +22,8 @@ struct ImmutableCFOptions {
 
   CompactionStyle compaction_style;
 
+  CompactionPri compaction_pri;
+
   CompactionOptionsUniversal compaction_options_universal;
   CompactionOptionsFIFO compaction_options_fifo;
 
@@ -102,10 +104,12 @@ struct ImmutableCFOptions {
   std::vector<std::shared_ptr<EventListener>> listeners;
 
   std::shared_ptr<Cache> row_cache;
+
+  uint32_t max_subcompactions;
 };
 
 struct MutableCFOptions {
-  MutableCFOptions(const Options& options, const ImmutableCFOptions& ioptions)
+  explicit MutableCFOptions(const ColumnFamilyOptions& options)
       : write_buffer_size(options.write_buffer_size),
         max_write_buffer_number(options.max_write_buffer_number),
         arena_block_size(options.arena_block_size),
@@ -123,7 +127,6 @@ struct MutableCFOptions {
             options.level0_file_num_compaction_trigger),
         level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
         level0_stop_writes_trigger(options.level0_stop_writes_trigger),
-        compaction_pri(options.compaction_pri),
         max_compaction_bytes(options.max_compaction_bytes),
         target_file_size_base(options.target_file_size_base),
         target_file_size_multiplier(options.target_file_size_multiplier),
@@ -132,16 +135,15 @@ struct MutableCFOptions {
         max_bytes_for_level_multiplier_additional(
             options.max_bytes_for_level_multiplier_additional),
         verify_checksums_in_compaction(options.verify_checksums_in_compaction),
-        max_subcompactions(options.max_subcompactions),
         max_sequential_skip_in_iterations(
             options.max_sequential_skip_in_iterations),
         paranoid_file_checks(options.paranoid_file_checks),
         report_bg_io_stats(options.report_bg_io_stats),
         compression(options.compression),
-        min_partial_merge_operands(options.min_partial_merge_operands),
+        min_partial_merge_operands(options.min_partial_merge_operands) {
-        compaction_options_fifo(ioptions.compaction_options_fifo) {
+    RefreshDerivedOptions(options.num_levels, options.compaction_style);
-    RefreshDerivedOptions(ioptions);
   }
+
   MutableCFOptions()
       : write_buffer_size(0),
         max_write_buffer_number(0),
@@ -156,14 +158,12 @@ struct MutableCFOptions {
         level0_file_num_compaction_trigger(0),
         level0_slowdown_writes_trigger(0),
         level0_stop_writes_trigger(0),
-        compaction_pri(kByCompensatedSize),
         max_compaction_bytes(0),
         target_file_size_base(0),
         target_file_size_multiplier(0),
         max_bytes_for_level_base(0),
         max_bytes_for_level_multiplier(0),
         verify_checksums_in_compaction(false),
-        max_subcompactions(1),
         max_sequential_skip_in_iterations(0),
         paranoid_file_checks(false),
         report_bg_io_stats(false),
@@ -171,7 +171,11 @@ struct MutableCFOptions {
         min_partial_merge_operands(2) {}
 
   // Must be called after any change to MutableCFOptions
-  void RefreshDerivedOptions(const ImmutableCFOptions& ioptions);
+  void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style);
+
+  void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) {
+    RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style);
+  }
 
   // Get the max file size in a given level.
   uint64_t MaxFileSizeForLevel(int level) const;
@@ -201,7 +205,6 @@ struct MutableCFOptions {
   int level0_file_num_compaction_trigger;
   int level0_slowdown_writes_trigger;
   int level0_stop_writes_trigger;
-  CompactionPri compaction_pri;
   uint64_t max_compaction_bytes;
   uint64_t target_file_size_base;
   int target_file_size_multiplier;
@@ -209,7 +212,6 @@ struct MutableCFOptions {
   int max_bytes_for_level_multiplier;
   std::vector<int> max_bytes_for_level_multiplier_additional;
   bool verify_checksums_in_compaction;
-  int max_subcompactions;
 
   // Misc options
   uint64_t max_sequential_skip_in_iterations;
@@ -217,7 +219,6 @@ struct MutableCFOptions {
   bool report_bg_io_stats;
   CompressionType compression;
   uint32_t min_partial_merge_operands;
-  CompactionOptionsFIFO compaction_options_fifo;
 
   // Derived options
   // Per-level target file size.