Change `bottommost_temperture` to `last_level_temperture` (#10471)

Summary:
Change tiered compaction feature from `bottommost_temperture` to
`last_level_temperture`. The old option is kept for migration purpose only,
which is behaving the same as `last_level_temperture` and it will be removed in
the next release.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10471

Test Plan: CI

Reviewed By: siying

Differential Revision: D38450621

Pulled By: jay-zhuang

fbshipit-source-id: cc1cdf8bad409376fec0152abc0a64fb72a91527

HISTORY.md

@@ -11,6 +11,7 @@
 ### Public API changes
 * Removed Customizable support for RateLimiter and removed its CreateFromString() and Type() functions.
 * `CompactRangeOptions::exclusive_manual_compaction` is now false by default. This ensures RocksDB does not introduce artificial parallelism limitations by default.
+* Tiered Storage: change `bottommost_temperture` to `last_level_temperture`. The old option name is kept only for migration, please use the new option. The behavior is changed to apply temperature for the `last_level` SST files only.
 
 ### Bug Fixes
 * Fix a bug starting in 7.4.0 in which some fsync operations might be skipped in a DB after any DropColumnFamily on that DB, until it is re-opened. This can lead to data loss on power loss. (For custom FileSystem implementations, this could lead to `FSDirectory::Fsync` or `FSDirectory::Close` after the first `FSDirectory::Close`; Also, valgrind could report call to `close()` with `fd=-1`.)

db/compaction/compaction.h

@@ -212,6 +212,11 @@ class Compaction {
   // Is this compaction creating a file in the bottom most level?
   bool bottommost_level() const { return bottommost_level_; }
 
+  // Is the compaction compact to the last level
+  bool is_last_level() const {
+    return output_level_ == immutable_options_.num_levels - 1;
+  }
+
   // Does this compaction include all sst files?
   bool is_full_compaction() const { return is_full_compaction_; }
 

db/compaction/compaction_iterator.cc

@@ -1107,13 +1107,13 @@ void CompactionIterator::DecideOutputLevel() {
         compaction_->WithinPenultimateLevelOutputRange(ikey_.user_key);
     if (!safe_to_penultimate_level) {
       output_to_penultimate_level_ = false;
-      // It could happen when disable/enable `bottommost_temperature` while
-      // holding a snapshot. When `bottommost_temperature` is not set
+      // It could happen when disable/enable `last_level_temperature` while
+      // holding a snapshot. When `last_level_temperature` is not set
       // (==kUnknown), the data newer than any snapshot is pushed to the last
       // level, but when the per_key_placement feature is enabled on the fly,
       // the data later than the snapshot has to be moved to the penultimate
       // level, which may or may not be safe. So the user needs to make sure all
-      // snapshot is released before enabling `bottommost_temperature` feature
+      // snapshot is released before enabling `last_level_temperature` feature
       // We will migrate the feature to `last_level_temperature` and maybe make
       // it not dynamically changeable.
       if (ikey_.sequence > earliest_snapshot_) {

db/compaction/compaction_job.cc

@@ -1716,13 +1716,16 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
                            &syncpoint_arg);
 #endif
 
-  // Pass temperature of botommost files to FileSystem.
+  // Pass temperature of the last level files to FileSystem.
   FileOptions fo_copy = file_options_;
   Temperature temperature = sub_compact->compaction->output_temperature();
-  if (temperature == Temperature::kUnknown && bottommost_level_ &&
+  // only set for the last level compaction and also it's not output to
+  // penultimate level (when preclude_last_level feature is enabled)
+  if (temperature == Temperature::kUnknown &&
+      sub_compact->compaction->is_last_level() &&
       !sub_compact->IsCurrentPenultimateLevel()) {
     temperature =
-        sub_compact->compaction->mutable_cf_options()->bottommost_temperature;
+        sub_compact->compaction->mutable_cf_options()->last_level_temperature;
   }
   fo_copy.temperature = temperature;
 

db/compaction/tiered_compaction_test.cc

@@ -16,7 +16,8 @@ namespace ROCKSDB_NAMESPACE {
 
 #if !defined(ROCKSDB_LITE)
 
-class TieredCompactionTest : public DBTestBase {
+class TieredCompactionTest : public DBTestBase,
+                             public testing::WithParamInterface<bool> {
  public:
   TieredCompactionTest()
       : DBTestBase("tiered_compaction_test", /*env_do_fsync=*/true),
@@ -120,6 +121,16 @@ class TieredCompactionTest : public DBTestBase {
     pl_stats.Clear();
   }
 
+  // bottommost_temperature is renaming to last_level_temperature, set either
+  // of them should have the same effect.
+  void SetColdTemperature(Options& options) {
+    if (GetParam()) {
+      options.bottommost_temperature = Temperature::kCold;
+    } else {
+      options.last_level_temperature = Temperature::kCold;
+    }
+  }
+
  private:
   void CompareStats(uint64_t val, uint64_t expect) {
     if (expect > 0) {
@@ -159,7 +170,7 @@ class TieredCompactionTest : public DBTestBase {
   }
 };
 
-TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
+TEST_P(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
@@ -167,7 +178,7 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
 
   auto options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.statistics = CreateDBStatistics();
   options.max_subcompactions = 10;
@@ -321,7 +332,7 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
+TEST_P(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
@@ -329,7 +340,7 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
 
   auto options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.statistics = CreateDBStatistics();
   options.max_subcompactions = 10;
@@ -513,13 +524,14 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
       1);
 }
 
-TEST_F(TieredCompactionTest, LevelColdRangeDelete) {
+TEST_P(TieredCompactionTest, LevelColdRangeDelete) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
+  const int kLastLevel = kNumLevels - 1;
 
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
@@ -544,11 +556,13 @@ TEST_F(TieredCompactionTest, LevelColdRangeDelete) {
   CompactRangeOptions cro;
   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
-  ASSERT_EQ("0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  ASSERT_EQ("0,1",
+            FilesPerLevel());  // bottommost but not last level file is hot
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
-  MoveFilesToLevel(kNumLevels - 1);
+  // explicitly move the data to the last level
+  MoveFilesToLevel(kLastLevel);
 
   ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
 
@@ -616,14 +630,14 @@ class SingleKeySstPartitionerFactory : public SstPartitionerFactory {
   }
 };
 
-TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
+TEST_P(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
   const int kNumTrigger = 4;
   const int kNumLevels = 3;
   const int kNumKeys = 10;
 
   auto factory = std::make_shared<SingleKeySstPartitionerFactory>();
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
@@ -734,7 +748,7 @@ TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, UniversalRangeDelete) {
+TEST_P(TieredCompactionTest, UniversalRangeDelete) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 10;
@@ -743,7 +757,7 @@ TEST_F(TieredCompactionTest, UniversalRangeDelete) {
 
   auto options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.statistics = CreateDBStatistics();
   options.sst_partitioner_factory = factory;
@@ -866,14 +880,14 @@ TEST_F(TieredCompactionTest, UniversalRangeDelete) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
+TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
   const int kLastLevel = kNumLevels - 1;
 
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
@@ -904,23 +918,22 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   }
   ASSERT_OK(dbfull()->WaitForCompact(true));
 
-  // non-last-level compaction doesn't support per_key_placement
+  // non last level is hot
   ASSERT_EQ("0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
   expect_stats[1].Add(kBasicCompStats);
   expect_stats[1].Add(kBasicPerLevelStats);
   expect_stats[1].ResetCompactionReason(CompactionReason::kLevelL0FilesNum);
   VerifyCompactionStats(expect_stats, expect_pl_stats);
 
+  // move all data to the last level
   MoveFilesToLevel(kLastLevel);
 
   ResetAllStats(expect_stats, expect_pl_stats);
 
-  // the data should be all hot, and it's a last level compaction, but all
-  // sequence numbers have been zeroed out, so they're still treated as old
-  // data.
+  // The compaction won't move the data up
   CompactRangeOptions cro;
   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
@@ -973,49 +986,19 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
 
   // move forward the cold_seq, try to split the data into cold and hot, but in
   // this case it's unsafe to split the data
+  // because it's non-last-level but bottommost file, the sequence number will
+  // be zeroed out and lost the time information (with
+  // `level_compaction_dynamic_level_bytes` or Universal Compaction, it should
+  // be rare.)
+  // TODO(zjay): ideally we should avoid zero out non-last-level bottommost file
   latest_cold_seq = seq_history[1];
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
   ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
   seq_history.clear();
 
-  // Add new data again
-  for (int i = 0; i < kNumTrigger; i++) {
-    for (int j = 0; j < kNumKeys; j++) {
-      ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i)));
-    }
-    ASSERT_OK(Flush());
-    seq_history.emplace_back(dbfull()->GetLatestSequenceNumber());
-  }
-  ASSERT_OK(dbfull()->WaitForCompact(true));
-
-  ResetAllStats(expect_stats, expect_pl_stats);
-
-  // Try to split the last level cold data into hot and cold, which
-  // is not supported
-  latest_cold_seq = seq_history[0];
-  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
-  ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
-
-  auto comp_stats = kBasicCompStats;
-  comp_stats.ResetCompactionReason(CompactionReason::kManualCompaction);
-  const int bottommost_level = 5;
-  expect_stats[bottommost_level].Add(comp_stats);
-  expect_stats[bottommost_level].Add(
-      comp_stats);  // bottommost level has 2 compactions
-  expect_stats[bottommost_level].Add(kBasicPerLevelStats);
-  expect_stats[bottommost_level].bytes_read_output_level = kHasValue;
-  expect_stats[bottommost_level].num_input_files_in_output_level = kHasValue;
-
-  for (int level = 2; level < bottommost_level; level++) {
-    expect_stats[level].bytes_moved = kHasValue;
-  }
-  VerifyCompactionStats(expect_stats, expect_pl_stats);
-
   // manually move all data (cold) to last level
   MoveFilesToLevel(kLastLevel);
   seq_history.clear();
@@ -1097,9 +1080,6 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
 
   db_->ReleaseSnapshot(snap);
 
-  // TODO: it should push the data to last level, but penultimate level file is
-  //  already bottommost, it's a conflict between bottommost_temperature and
-  //  tiered compaction which only applies to last level compaction.
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
   ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
   ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
@@ -1124,13 +1104,13 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
+TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
 
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.level_compaction_dynamic_level_bytes = true;
   options.num_levels = kNumLevels;
@@ -1232,6 +1212,9 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
       1);
 }
 
+INSTANTIATE_TEST_CASE_P(TieredCompactionTest, TieredCompactionTest,
+                        testing::Bool());
+
 #endif  // !defined(ROCKSDB_LITE)
 
 }  // namespace ROCKSDB_NAMESPACE

db/db_test2.cc

@@ -6676,7 +6676,7 @@ TEST_P(RenameCurrentTest, Compaction) {
   ASSERT_EQ("d_value", Get("d"));
 }
 
-TEST_F(DBTest2, BottommostTemperature) {
+TEST_F(DBTest2, LastLevelTemperature) {
   class TestListener : public EventListener {
    public:
     void OnFileReadFinish(const FileOperationInfo& info) override {
@@ -6730,11 +6730,16 @@ TEST_F(DBTest2, BottommostTemperature) {
     port::Mutex mutex_;
   };
 
+  const int kNumLevels = 7;
+  const int kLastLevel = kNumLevels - 1;
+
   auto* listener = new TestListener();
 
   Options options = CurrentOptions();
   options.bottommost_temperature = Temperature::kWarm;
   options.level0_file_num_compaction_trigger = 2;
+  options.level_compaction_dynamic_level_bytes = true;
+  options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
   options.listeners.emplace_back(listener);
   Reopen(options);
@@ -6760,7 +6765,7 @@ TEST_F(DBTest2, BottommostTemperature) {
   ColumnFamilyMetaData metadata;
   db_->GetColumnFamilyMetaData(&metadata);
   ASSERT_EQ(1, metadata.file_count);
-  SstFileMetaData meta = metadata.levels[1].files[0];
+  SstFileMetaData meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   uint64_t number;
   FileType type;
@@ -6818,7 +6823,7 @@ TEST_F(DBTest2, BottommostTemperature) {
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 
-  meta = metadata.levels[1].files[0];
+  meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
@@ -6837,7 +6842,7 @@ TEST_F(DBTest2, BottommostTemperature) {
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 
-  meta = metadata.levels[1].files[0];
+  meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
@@ -6865,13 +6870,13 @@ TEST_F(DBTest2, BottommostTemperature) {
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 
-  meta = metadata.levels[1].files[0];
+  meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 }
 
-TEST_F(DBTest2, BottommostTemperatureUniversal) {
+TEST_F(DBTest2, LastLevelTemperatureUniversal) {
   const int kTriggerNum = 3;
   const int kNumLevels = 5;
   const int kBottommostLevel = kNumLevels - 1;
@@ -6997,7 +7002,7 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
   ASSERT_EQ(std::atoi(prop.c_str()), 0);
 
   // Update bottommost temperature dynamically with SetOptions
-  auto s = db_->SetOptions({{"bottommost_temperature", "kCold"}});
+  auto s = db_->SetOptions({{"last_level_temperature", "kCold"}});
   ASSERT_OK(s);
   ASSERT_EQ(db_->GetOptions().bottommost_temperature, Temperature::kCold);
   db_->GetColumnFamilyMetaData(&metadata);
@@ -7097,6 +7102,9 @@ TEST_F(DBTest2, CheckpointFileTemperature) {
   std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
   Options options = CurrentOptions();
   options.bottommost_temperature = Temperature::kWarm;
+  // set dynamic_level to true so the compaction would compact the data to the
+  // last level directly which will have the last_level_temperature
+  options.level_compaction_dynamic_level_bytes = true;
   options.level0_file_num_compaction_trigger = 2;
   options.env = env.get();
   Reopen(options);
@@ -7153,6 +7161,9 @@ TEST_F(DBTest2, FileTemperatureManifestFixup) {
   std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
   Options options = CurrentOptions();
   options.bottommost_temperature = Temperature::kWarm;
+  // set dynamic_level to true so the compaction would compact the data to the
+  // last level directly which will have the last_level_temperature
+  options.level_compaction_dynamic_level_bytes = true;
   options.level0_file_num_compaction_trigger = 2;
   options.env = env.get();
   std::vector<std::string> cfs = {/*"default",*/ "test1", "test2"};

db/seqno_time_test.cc

@@ -232,13 +232,9 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
     }
     ASSERT_OK(Flush());
   }
-  // TODO(zjay): all data become cold because of level 5 (penultimate level) is
-  //  the bottommost level, which converts the data to cold. PerKeyPlacement is
-  //  for the last level (level 6). Will be fixed by change the
-  //  bottommost_temperature to the last_level_temperature
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
   // Compact the files to the last level which should split the hot/cold data
   MoveFilesToLevel(6);
@@ -249,7 +245,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
   // the first a few key should be cold
   AssertKetTemperature(20, Temperature::kCold);
 
-  // Wait some time, each it wait, the cold data is increasing and hot data is
+  // Wait some time, with each wait, the cold data is increasing and hot data is
   // decreasing
   for (int i = 0; i < 30; i++) {
     dbfull()->TEST_WaitForPeridicWorkerRun(

include/rocksdb/advanced_options.h

@@ -872,12 +872,19 @@ struct AdvancedColumnFamilyOptions {
 
   // EXPERIMENTAL
   // The feature is still in development and is incomplete.
-  // If this option is set, when creating bottommost files, pass this
+  // If this option is set, when creating the last level files, pass this
   // temperature to FileSystem used. Should be no-op for default FileSystem
   // and users need to plug in their own FileSystem to take advantage of it.
   //
+  // Note: the feature is changed from `bottommost_temperature` to
+  //  `last_level_temperature` which now only apply for the last level files.
+  //  The option name `bottommost_temperature` is kept only for migration, the
+  //  behavior is the same as `last_level_temperature`. Please stop using
+  //  `bottommost_temperature` and will be removed in next release.
+  //
   // Dynamically changeable through the SetOptions() API
   Temperature bottommost_temperature = Temperature::kUnknown;
+  Temperature last_level_temperature = Temperature::kUnknown;
 
   // EXPERIMENTAL
   // The feature is still in development and is incomplete.

options/cf_options.cc

@@ -411,7 +411,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
           OptionType::kUInt64T, OptionVerificationType::kNormal,
           OptionTypeFlags::kMutable}},
         {"bottommost_temperature",
-         {offsetof(struct MutableCFOptions, bottommost_temperature),
+         {0, OptionType::kTemperature, OptionVerificationType::kDeprecated,
+          OptionTypeFlags::kMutable}},
+        {"last_level_temperature",
+         {offsetof(struct MutableCFOptions, last_level_temperature),
           OptionType::kTemperature, OptionVerificationType::kNormal,
           OptionTypeFlags::kMutable}},
         {"enable_blob_files",
@@ -1105,8 +1108,8 @@ void MutableCFOptions::Dump(Logger* log) const {
                  prepopulate_blob_cache == PrepopulateBlobCache::kFlushOnly
                      ? "flush only"
                      : "disable");
-  ROCKS_LOG_INFO(log, "                   bottommost_temperature: %d",
-                 static_cast<int>(bottommost_temperature));
+  ROCKS_LOG_INFO(log, "                   last_level_temperature: %d",
+                 static_cast<int>(last_level_temperature));
 }
 
 MutableCFOptions::MutableCFOptions(const Options& options)

options/cf_options.h

@@ -158,7 +158,10 @@ struct MutableCFOptions {
         bottommost_compression(options.bottommost_compression),
         compression_opts(options.compression_opts),
         bottommost_compression_opts(options.bottommost_compression_opts),
-        bottommost_temperature(options.bottommost_temperature),
+        last_level_temperature(options.last_level_temperature ==
+                                       Temperature::kUnknown
+                                   ? options.bottommost_temperature
+                                   : options.last_level_temperature),
         sample_for_compression(
             options.sample_for_compression),  // TODO: is 0 fine here?
         compression_per_level(options.compression_per_level) {
@@ -206,7 +209,7 @@ struct MutableCFOptions {
         report_bg_io_stats(false),
         compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
         bottommost_compression(kDisableCompressionOption),
-        bottommost_temperature(Temperature::kUnknown),
+        last_level_temperature(Temperature::kUnknown),
         sample_for_compression(0) {}
 
   explicit MutableCFOptions(const Options& options);
@@ -294,9 +297,7 @@ struct MutableCFOptions {
   CompressionType bottommost_compression;
   CompressionOptions compression_opts;
   CompressionOptions bottommost_compression_opts;
-  // TODO this experimental option isn't made configurable
-  // through strings yet.
-  Temperature bottommost_temperature;
+  Temperature last_level_temperature;
 
   uint64_t sample_for_compression;
   std::vector<CompressionType> compression_per_level;

options/options_helper.cc

@@ -272,7 +272,8 @@ void UpdateColumnFamilyOptions(const MutableCFOptions& moptions,
   cf_opts->bottommost_compression_opts = moptions.bottommost_compression_opts;
   cf_opts->sample_for_compression = moptions.sample_for_compression;
   cf_opts->compression_per_level = moptions.compression_per_level;
-  cf_opts->bottommost_temperature = moptions.bottommost_temperature;
+  cf_opts->last_level_temperature = moptions.last_level_temperature;
+  cf_opts->bottommost_temperature = moptions.last_level_temperature;
 }
 
 void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions,

options/options_settable_test.cc

@@ -528,6 +528,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
       "blob_file_starting_level=1;"
       "prepopulate_blob_cache=kDisable;"
       "bottommost_temperature=kWarm;"
+      "last_level_temperature=kWarm;"
       "preclude_last_level_data_seconds=86400;"
       "compaction_options_fifo={max_table_files_size=3;allow_"
       "compaction=false;age_for_warm=1;};"

options/options_test.cc

@@ -128,7 +128,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
       {"blob_compaction_readahead_size", "256K"},
       {"blob_file_starting_level", "1"},
       {"prepopulate_blob_cache", "kDisable"},
-      {"bottommost_temperature", "kWarm"},
+      {"last_level_temperature", "kWarm"},
   };
 
   std::unordered_map<std::string, std::string> db_options_map = {
@@ -268,6 +268,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
   ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1);
   ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable);
+  ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm);
   ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
 
   cf_options_map["write_buffer_size"] = "hello";
@@ -2359,7 +2360,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
       {"blob_compaction_readahead_size", "256K"},
       {"blob_file_starting_level", "1"},
       {"prepopulate_blob_cache", "kDisable"},
-      {"bottommost_temperature", "kWarm"},
+      {"last_level_temperature", "kWarm"},
   };
 
   std::unordered_map<std::string, std::string> db_options_map = {
@@ -2493,6 +2494,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
   ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1);
   ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable);
+  ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm);
   ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
 
   cf_options_map["write_buffer_size"] = "hello";

tools/db_bench_tool.cc

@@ -1309,7 +1309,7 @@ DEFINE_string(fs_uri, "",
 DEFINE_string(simulate_hybrid_fs_file, "",
               "File for Store Metadata for Simulate hybrid FS. Empty means "
               "disable the feature. Now, if it is set, "
-              "bottommost_temperature is set to kWarm.");
+              "last_level_temperature is set to kWarm.");
 DEFINE_int32(simulate_hybrid_hdd_multipliers, 1,
              "In simulate_hybrid_fs_file or simulate_hdd mode, how many HDDs "
              "are simulated.");

utilities/backup/backup_engine_test.cc

@@ -4046,6 +4046,9 @@ TEST_F(BackupEngineTest, FileTemperatures) {
   // Use temperatures
   options_.bottommost_temperature = Temperature::kWarm;
   options_.level0_file_num_compaction_trigger = 2;
+  // set dynamic_level to true so the compaction would compact the data to the
+  // last level directly which will have the last_level_temperature
+  options_.level_compaction_dynamic_level_bytes = true;
 
   OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
                         kShareWithChecksum);