Bottommost level compaction option

Summary: Replace force_bottommost_level_compaction in CompactRangeOption with an option that allow the user to (always skip, always compact, compact if compaction filter is present) the bottommost level for level based compaction.

Test Plan: make check

Reviewers: sdong, yhchiang, igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D40527

HISTORY.md

@@ -18,7 +18,7 @@
 * When options.soft_rate_limit or options.level0_slowdown_writes_trigger is triggered, the way to slow down writes is changed to: write rate to DB is limited to to options.delayed_write_rate.
 * DB::GetApproximateSizes() adds a parameter to allow the estimation to include data in mem table, with default to be not to include. It is now only supported in skip list mem table.
 * DB::CompactRange() now accept CompactRangeOptions instead of multiple paramters. CompactRangeOptions is defined in include/rocksdb/options.h.
-* Add force_bottommost_level_compaction option to CompactRangeOptions, which prevent compaction from skipping compacting bottommost level.
+* CompactRange() will now skip bottommost level compaction for level based compaction if there is no compaction filter, bottommost_level_compaction is introduced in CompactRangeOptions to control when it's possbile to skip bottommost level compaction. This mean that if you want the compaction to produce a single file you need to set bottommost_level_compaction to BottommostLevelCompaction::kForce.
 * Add Cache.GetPinnedUsage() to get the size of memory occupied by entries that are in use by the system.
 * DB:Open() will fail if the compression specified in Options is not linked with the binary. If you see this failure, recompile RocksDB with compression libraries present on your system. Also, previously our default compression was snappy. This behavior is now changed. Now, the default compression is snappy only if it's available on the system. If it isn't we change the default to kNoCompression.
 

db/db_impl.cc

@@ -1429,13 +1429,17 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
           cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
         output_level = level;
       } else if (level == max_level_with_files && level > 0) {
-        if (options.force_bottommost_level_compaction == false &&
-            cfd->ioptions()->compaction_filter == nullptr &&
-            cfd->ioptions()->compaction_filter_factory == nullptr &&
-            cfd->ioptions()->compaction_filter_factory_v2 == nullptr) {
-          // If we are not forced to compact the bottommost level and there is
-          // no compaction filter we can skip the compaction of
-          // the bottommost level
+        if (options.bottommost_level_compaction ==
+            BottommostLevelCompaction::kSkip) {
+          // Skip bottommost level compaction
+          continue;
+        } else if (options.bottommost_level_compaction ==
+                       BottommostLevelCompaction::kIfHaveCompactionFilter &&
+                   cfd->ioptions()->compaction_filter == nullptr &&
+                   cfd->ioptions()->compaction_filter_factory == nullptr &&
+                   cfd->ioptions()->compaction_filter_factory_v2 == nullptr) {
+          // Skip bottommost level compaction since we dont have
+          // compaction filter
           continue;
         }
         output_level = level;

db/db_test.cc

@@ -5438,7 +5438,8 @@ TEST_F(DBTest, ConvertCompactionStyle) {
   CompactRangeOptions compact_options;
   compact_options.change_level = true;
   compact_options.target_level = 0;
-  compact_options.force_bottommost_level_compaction = true;
+  compact_options.bottommost_level_compaction =
+      BottommostLevelCompaction::kForce;
   dbfull()->CompactRange(compact_options, handles_[1], nullptr, nullptr);
 
   // Only 1 file in L0
@@ -13989,13 +13990,34 @@ TEST_F(DBTest, ForceBottommostLevelCompaction) {
   // Compaction will do L0=>L1 L1=>L2 L2=>L3 (3 trivial moves)
   // then compacte the bottommost level L3=>L3 (non trivial move)
   compact_options = CompactRangeOptions();
-  compact_options.force_bottommost_level_compaction = true;
+  compact_options.bottommost_level_compaction =
+      BottommostLevelCompaction::kForce;
   ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
   ASSERT_EQ("0,0,0,1", FilesPerLevel(0));
   ASSERT_EQ(trivial_move, 4);
   ASSERT_EQ(non_trivial_move, 1);
 
-  for (int i = 0; i < 200; i++) {
+  // File with keys [ 200 => 299 ]
+  for (int i = 200; i < 300; i++) {
+    values.push_back(RandomString(&rnd, value_size));
+    ASSERT_OK(Put(Key(i), values[i]));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_EQ("1,0,0,1", FilesPerLevel(0));
+  trivial_move = 0;
+  non_trivial_move = 0;
+  compact_options = CompactRangeOptions();
+  compact_options.bottommost_level_compaction =
+      BottommostLevelCompaction::kSkip;
+  // Compaction will do L0=>L1 L1=>L2 L2=>L3 (3 trivial moves)
+  // and will skip bottommost level compaction
+  ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
+  ASSERT_EQ("0,0,0,2", FilesPerLevel(0));
+  ASSERT_EQ(trivial_move, 3);
+  ASSERT_EQ(non_trivial_move, 0);
+
+  for (int i = 0; i < 300; i++) {
     ASSERT_EQ(Get(Key(i)), values[i]);
   }
 

include/rocksdb/options.h

@@ -1266,6 +1266,18 @@ struct CompactionOptions {
         output_file_size_limit(std::numeric_limits<uint64_t>::max()) {}
 };
 
+// For level based compaction, we can configure if we want to skip/force
+// bottommost level compaction.
+enum class BottommostLevelCompaction {
+  // Skip bottommost level compaction
+  kSkip,
+  // Only compact bottommost level if there is a compaction filter
+  // This is the default option
+  kIfHaveCompactionFilter,
+  // Always compact bottommost level
+  kForce,
+};
+
 // CompactRangeOptions is used by CompactRange() call.
 struct CompactRangeOptions {
   // If true, compacted files will be moved to the minimum level capable
@@ -1277,10 +1289,10 @@ struct CompactRangeOptions {
   // Compaction outputs will be placed in options.db_paths[target_path_id].
   // Behavior is undefined if target_path_id is out of range.
   uint32_t target_path_id = 0;
-  // By default compaction will try to skip compacting bottommost level if
-  // possible, setting this flag to true will force compaction to compact
-  // the bottomost level.
-  bool force_bottommost_level_compaction = false;
+  // By default level based compaction will only compact the bottommost level
+  // if there is a compaction filter
+  BottommostLevelCompaction bottommost_level_compaction =
+      BottommostLevelCompaction::kIfHaveCompactionFilter;
 };
 }  // namespace rocksdb