Add an unittest for Periodic compaction conflict with ongoing compaction (#10908)

Summary: Add a tiered storage migration test which would conflict with an ongoing penultimate level compaction. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10908 Test Plan: Test only change Reviewed By: anand1976 Differential Revision: D40864509 Pulled By: ajkr fbshipit-source-id: e316e849a01a6c71a41be130101f909b6c0498cb
2 years ago · 1078d860a9
parent 4d60cbc629
commit 1078d860a9
6 changed files with 211 additions and 84 deletions
--- a/db/compaction/tiered_compaction_test.cc
+++ b/db/compaction/tiered_compaction_test.cc
@ -1251,7 +1251,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });

  int sst_num = 0;
@ -1259,7 +1259,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
  for (; sst_num < kNumTrigger; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -1313,7 +1313,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });

  int sst_num = 0;
@ -1321,7 +1321,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
  for (; sst_num < kNumTrigger; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -1355,7 +1355,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
    for (int i = 0; i < kNumKeys; i++) {
      // the value needs to be big enough to trigger full compaction
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100)));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -1389,7 +1389,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });

  int sst_num = 0;
@ -1397,7 +1397,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
  for (; sst_num < kNumTrigger; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -1463,13 +1463,13 @@ TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) {

  Random rnd(301);

-  dbfull()->TEST_WaitForPeridicTaskRun([&] {
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
    mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(10) + 1));
  });

  for (int i = 0; i < kNumKeys; i++) {
    ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
-    dbfull()->TEST_WaitForPeridicTaskRun([&] {
+    dbfull()->TEST_WaitForPeriodicTaskRun([&] {
      mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(2)));
    });
  }
@ -1516,7 +1516,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });

  int sst_num = 0;
@ -1524,7 +1524,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
  for (; sst_num < kNumTrigger; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -1594,7 +1594,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });

  Random rnd(301);
@ -1603,7 +1603,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
  for (; sst_num < kNumTrigger; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100)));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -1696,7 +1696,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
    for (int i = 0; i < kNumKeys; i++) {
      // the value needs to be big enough to trigger full compaction
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -1721,6 +1721,133 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
  Close();
 }

+TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
+  // Test the last level only periodic compaction should also be blocked by an
+  // ongoing compaction in penultimate level if tiered compaction is enabled
+  // otherwise, the periodic compaction should just run for the last level.
+  const int kNumTrigger = 4;
+  const int kNumLevels = 7;
+  const int kPenultimateLevel = kNumLevels - 2;
+  const int kKeyPerSec = 1;
+  const int kNumKeys = 100;
+
+  bool enable_preclude_last_level = GetParam();
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.preserve_internal_time_seconds = 20000;
+  options.env = mock_env_.get();
+  options.level0_file_num_compaction_trigger = kNumTrigger;
+  options.num_levels = kNumLevels;
+  options.ignore_max_compaction_bytes_for_input = false;
+  options.periodic_compaction_seconds = 10000;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+
+  for (int i = 0; i < 3 * kNumKeys; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
+  }
+  ASSERT_OK(Flush());
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+
+  // make sure all data is compacted to the last level
+  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
+
+  // enable preclude feature
+  if (enable_preclude_last_level) {
+    options.preclude_last_level_data_seconds = 20000;
+  }
+  options.max_background_jobs = 8;
+  options.last_level_temperature = Temperature::kCold;
+  Reopen(options);
+
+  std::atomic_bool is_size_ratio_compaction_running = false;
+  std::atomic_bool verified_last_level_compaction = false;
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) {
+        auto compaction = static_cast<Compaction*>(arg);
+        if (compaction->output_level() == kPenultimateLevel) {
+          is_size_ratio_compaction_running = true;
+          TEST_SYNC_POINT(
+              "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+              "SizeRatioCompaction1");
+          TEST_SYNC_POINT(
+              "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+              "SizeRatioCompaction2");
+          is_size_ratio_compaction_running = false;
+        }
+      });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) {
+        auto compaction = static_cast<Compaction*>(arg);
+
+        if (is_size_ratio_compaction_running) {
+          if (enable_preclude_last_level) {
+            ASSERT_TRUE(compaction == nullptr);
+          } else {
+            ASSERT_TRUE(compaction != nullptr);
+            ASSERT_EQ(compaction->compaction_reason(),
+                      CompactionReason::kPeriodicCompaction);
+            ASSERT_EQ(compaction->start_level(), kNumLevels - 1);
+          }
+          verified_last_level_compaction = true;
+        }
+        TEST_SYNC_POINT(
+            "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+            "AutoCompactionPicked");
+      });
+
+  SyncPoint::GetInstance()->LoadDependency({
+      {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+       "SizeRatioCompaction1",
+       "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite"},
+      {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+       "AutoCompactionPicked",
+       "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+       "SizeRatioCompaction2"},
+  });
+
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
+
+  for (int i = 0; i < kNumTrigger - 1; i++) {
+    for (int j = 0; j < kNumKeys; j++) {
+      ASSERT_OK(Put(Key(i * (kNumKeys - 1) + i), rnd.RandomString(10)));
+      dbfull()->TEST_WaitForPeriodicTaskRun(
+          [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
+    }
+    ASSERT_OK(Flush());
+  }
+
+  TEST_SYNC_POINT(
+      "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite");
+
+  // wait for periodic compaction time and flush to trigger the periodic
+  // compaction, which should be blocked by ongoing compaction in the
+  // penultimate level
+  mock_clock_->MockSleepForSeconds(10000);
+  for (int i = 0; i < 3 * kNumKeys; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(10)));
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(dbfull()->WaitForCompact(true));
+
+  stop_token.reset();
+
+  Close();
+}
+
 INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTestWithParms,
                        PrecludeLastLevelTestWithParms, testing::Bool());

@ -1781,14 +1908,14 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });

  Random rnd(301);

  for (int i = 0; i < 300; i++) {
    ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
  }
  ASSERT_OK(Flush());
@ -1892,7 +2019,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun([&] {
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
    mock_clock_->MockSleepForSeconds(static_cast<int>(kSecondsPerKey));
  });

@ -1917,13 +2044,13 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  Random rnd(301);
  for (int i = 0; i < kNumKeys; i++) {
    ASSERT_OK(Put(Key(i + 3), rnd.RandomString(kValueBytes)));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
  }
  auto* snap1 = db_->GetSnapshot();
  for (int i = 0; i < kNumKeys; i++) {
    ASSERT_OK(Put(Key(i), rnd.RandomString(kValueBytes)));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
  }
  auto* snap2 = db_->GetSnapshot();
@ -1937,7 +2064,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
                             Key(2 * kNumKeysPerFile - 1),
                             Key(2 * kNumKeysPerFile + 1)));
  ASSERT_OK(Flush());
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
  verify_db();

--- a/db/db_impl/db_impl.h
+++ b/db/db_impl/db_impl.h
@ -1161,7 +1161,7 @@ class DBImpl : public DB {
  int TEST_BGCompactionsAllowed() const;
  int TEST_BGFlushesAllowed() const;
  size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
-  void TEST_WaitForPeridicTaskRun(std::function<void()> callback) const;
+  void TEST_WaitForPeriodicTaskRun(std::function<void()> callback) const;
  SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const;
  size_t TEST_EstimateInMemoryStatsHistorySize() const;

--- a/db/db_impl/db_impl_debug.cc
+++ b/db/db_impl/db_impl_debug.cc
@ -290,7 +290,7 @@ size_t DBImpl::TEST_GetWalPreallocateBlockSize(
 }

 #ifndef ROCKSDB_LITE
-void DBImpl::TEST_WaitForPeridicTaskRun(std::function<void()> callback) const {
+void DBImpl::TEST_WaitForPeriodicTaskRun(std::function<void()> callback) const {
  periodic_task_scheduler_.TEST_WaitForRun(callback);
 }

--- a/db/periodic_task_scheduler_test.cc
+++ b/db/periodic_task_scheduler_test.cc
@ -65,7 +65,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
  ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec);

  ASSERT_GT(kPeriodSec, 1u);
-  dbfull()->TEST_WaitForPeridicTaskRun([&] {
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
    mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec) - 1);
  });

@ -77,14 +77,14 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
  ASSERT_EQ(1, pst_st_counter);
  ASSERT_EQ(1, flush_info_log_counter);

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });

  ASSERT_EQ(2, dump_st_counter);
  ASSERT_EQ(2, pst_st_counter);
  ASSERT_EQ(2, flush_info_log_counter);

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });

  ASSERT_EQ(3, dump_st_counter);
@ -98,7 +98,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
  ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec);

  // Info log flush should still run.
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
  ASSERT_EQ(3, dump_st_counter);
  ASSERT_EQ(3, pst_st_counter);
@ -113,7 +113,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {

  ASSERT_EQ(2, scheduler.TEST_GetValidTaskNum());

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
  ASSERT_EQ(4, dump_st_counter);
  ASSERT_EQ(3, pst_st_counter);
@ -154,19 +154,19 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
  ASSERT_EQ(kInstanceNum * 3, scheduler.TEST_GetValidTaskNum());

  int expected_run = kInstanceNum;
-  dbi->TEST_WaitForPeridicTaskRun(
+  dbi->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
  ASSERT_EQ(expected_run, dump_st_counter);
  ASSERT_EQ(expected_run, pst_st_counter);

  expected_run += kInstanceNum;
-  dbi->TEST_WaitForPeridicTaskRun(
+  dbi->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  ASSERT_EQ(expected_run, dump_st_counter);
  ASSERT_EQ(expected_run, pst_st_counter);

  expected_run += kInstanceNum;
-  dbi->TEST_WaitForPeridicTaskRun(
+  dbi->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  ASSERT_EQ(expected_run, dump_st_counter);
  ASSERT_EQ(expected_run, pst_st_counter);
@ -178,9 +178,9 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {

  expected_run += (kInstanceNum - half) * 2;

-  dbi->TEST_WaitForPeridicTaskRun(
+  dbi->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
-  dbi->TEST_WaitForPeridicTaskRun(
+  dbi->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  ASSERT_EQ(expected_run, dump_st_counter);
  ASSERT_EQ(expected_run, pst_st_counter);
--- a/db/seqno_time_test.cc
+++ b/db/seqno_time_test.cc
@ -80,7 +80,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });

  int sst_num = 0;
@ -88,7 +88,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
  for (; sst_num < kNumTrigger; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -110,7 +110,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
  for (; sst_num < kNumTrigger * 2; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -124,7 +124,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
  for (; sst_num < kNumTrigger * 3; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
        mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
      });
    }
@ -143,7 +143,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
  AssertKeyTemperature(20, Temperature::kCold);

  for (int i = 0; i < 30; i++) {
-    dbfull()->TEST_WaitForPeridicTaskRun([&] {
+    dbfull()->TEST_WaitForPeriodicTaskRun([&] {
      mock_clock_->MockSleepForSeconds(static_cast<int>(20 * kKeyPerSec));
    });
    ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
@ -161,7 +161,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
  // the compaction will not get the new seqno->time sampling to decide the last
  // a few data's time.
  for (int i = 0; i < 5; i++) {
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
    ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
  }
@ -192,7 +192,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {

  // pass some time first, otherwise the first a few keys write time are going
  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });

  int sst_num = 0;
@ -200,7 +200,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
  for (; sst_num < 4; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun(
+      dbfull()->TEST_WaitForPeriodicTaskRun(
          [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
    }
    ASSERT_OK(Flush());
@ -222,7 +222,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
  for (; sst_num < 14; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun(
+      dbfull()->TEST_WaitForPeriodicTaskRun(
          [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
    }
    ASSERT_OK(Flush());
@ -243,7 +243,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
  // Wait some time, with each wait, the cold data is increasing and hot data is
  // decreasing
  for (int i = 0; i < 30; i++) {
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); });
    ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
    uint64_t pre_hot = hot_data_size;
@ -262,7 +262,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
  // hot data might not be empty, because if we don't write new data, there's
  // no seqno->time sampling available to the compaction
  for (int i = 0; i < 5; i++) {
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
    ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
  }
@ -324,7 +324,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
  // Write a key every 10 seconds
  for (int i = 0; i < 200; i++) {
    ASSERT_OK(Put(Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
  }
  ASSERT_OK(Flush());
@ -357,7 +357,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
  // Write a key every 1 seconds
  for (int i = 0; i < 200; i++) {
    ASSERT_OK(Put(Key(i + 190), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1)); });
  }
  seq_end = dbfull()->GetLatestSequenceNumber();
@ -393,7 +393,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
  // Write a key every 200 seconds
  for (int i = 0; i < 200; i++) {
    ASSERT_OK(Put(Key(i + 380), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); });
  }
  seq_end = dbfull()->GetLatestSequenceNumber();
@ -435,7 +435,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
  // Write a key every 100 seconds
  for (int i = 0; i < 200; i++) {
    ASSERT_OK(Put(Key(i + 570), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
  }
  seq_end = dbfull()->GetLatestSequenceNumber();
@ -516,7 +516,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  // Write some data and increase the current time
  for (int i = 0; i < 200; i++) {
    ASSERT_OK(Put(Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
  }
  ASSERT_OK(Flush());
@ -536,7 +536,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  // Write some data to the default CF (without preclude_last_level feature)
  for (int i = 0; i < 200; i++) {
    ASSERT_OK(Put(Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
  }
  ASSERT_OK(Flush());
@ -544,7 +544,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  // Write some data to the CF one
  for (int i = 0; i < 20; i++) {
    ASSERT_OK(Put(1, Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
  }
  ASSERT_OK(Flush(1));
@ -568,7 +568,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  // Add more data to CF "two" to fill the in memory mapping
  for (int i = 0; i < 2000; i++) {
    ASSERT_OK(Put(2, Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
  }
  seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping();
@ -592,7 +592,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  // enabled have flushed, the in-memory seqno->time mapping should be cleared
  for (int i = 0; i < 10; i++) {
    ASSERT_OK(Put(0, Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
  }
  seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping();
@ -603,7 +603,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  for (int j = 0; j < 3; j++) {
    for (int i = 0; i < 200; i++) {
      ASSERT_OK(Put(2, Key(i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun(
+      dbfull()->TEST_WaitForPeriodicTaskRun(
          [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
    }
    ASSERT_OK(Flush(2));
@ -623,7 +623,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  for (int j = 0; j < 2; j++) {
    for (int i = 0; i < 200; i++) {
      ASSERT_OK(Put(0, Key(i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun(
+      dbfull()->TEST_WaitForPeriodicTaskRun(
          [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
    }
    ASSERT_OK(Flush(0));
@ -638,7 +638,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
  // Write some data to CF "two", but don't flush to accumulate
  for (int i = 0; i < 1000; i++) {
    ASSERT_OK(Put(2, Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
  }
  ASSERT_GE(
@ -682,7 +682,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiInstancesBasic) {
  WriteOptions wo;
  for (int i = 0; i < 200; i++) {
    ASSERT_OK(dbi->Put(wo, Key(i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
  }
  SeqnoToTimeMapping seqno_to_time_mapping = dbi->TEST_GetSeqnoToTimeMapping();
@ -720,7 +720,7 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) {
  for (; sst_num < kNumTrigger - 1; sst_num++) {
    for (int i = 0; i < kNumKeys; i++) {
      ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun(
+      dbfull()->TEST_WaitForPeriodicTaskRun(
          [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
    }
    ASSERT_OK(Flush());
@ -742,7 +742,7 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) {
  // Trigger a compaction
  for (int i = 0; i < kNumKeys; i++) {
    ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
  }
  sst_num++;
--- a/monitoring/stats_history_test.cc
+++ b/monitoring/stats_history_test.cc
@ -66,10 +66,10 @@ TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) {

  // Wait for the first stats persist to finish, as the initial delay could be
  // different.
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  ASSERT_GE(counter, 1);

@ -98,17 +98,17 @@ TEST_F(StatsHistoryTest, StatsPersistScheduling) {

  // Wait for the first stats persist to finish, as the initial delay could be
  // different.
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  ASSERT_GE(counter, 1);

  // Test cancel job through SetOptions
  ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
  int old_val = counter;
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec * 2); });
  ASSERT_EQ(counter, old_val);

@ -130,7 +130,7 @@ TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) {
      {{"stats_persist_period_sec", std::to_string(kPeriodSec)}}));
  ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec);

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  ASSERT_GE(counter, 1);
  Close();
@ -149,11 +149,11 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
  ReopenWithColumnFamilies({"default", "pikachu"}, options);

  // make sure the first stats persist to finish
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });

  // Wait for stats persist to finish
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });

  std::unique_ptr<StatsHistoryIterator> stats_iter;
@ -171,7 +171,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
  ASSERT_GT(stats_count, 0);
  // Wait a bit and verify no more stats are found
  for (int i = 0; i < 10; ++i) {
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(1); });
  }
  ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
@ -226,7 +226,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {

  const int kIterations = 10;
  for (int i = 0; i < kIterations; ++i) {
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  }

@ -250,7 +250,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {

  // Wait for stats persist to finish
  for (int i = 0; i < kIterations; ++i) {
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
        [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  }

@ -299,11 +299,11 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {

  // Wait for the first stats persist to finish, as the initial delay could be
  // different.
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });

  // Wait for stats persist to finish
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });

  auto iter =
@ -311,14 +311,14 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
  int key_count1 = countkeys(iter);
  delete iter;

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  iter =
      db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
  int key_count2 = countkeys(iter);
  delete iter;

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  iter =
      db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
@ -392,32 +392,32 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) {

  // Wait for the first stats persist to finish, as the initial delay could be
  // different.
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });

  // Wait for stats persist to finish
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  auto iter =
      db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
  countkeys(iter);
  delete iter;

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  iter =
      db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
  countkeys(iter);
  delete iter;

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  iter =
      db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
  countkeys(iter);
  delete iter;

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });

  std::map<std::string, uint64_t> stats_map_after;
@ -481,10 +481,10 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
  ASSERT_EQ(Get(2, "foo"), "bar");

  // make sure the first stats persist to finish
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  auto iter =
      db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
@ -581,7 +581,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {

  // Wait for the first stats persist to finish, as the initial delay could be
  // different.
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });

  ColumnFamilyData* cfd_default =
@ -600,7 +600,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
  ASSERT_OK(Put(1, "Eevee", "v0"));
  ASSERT_EQ("v0", Get(1, "Eevee"));

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  // writing to all three cf, flush default cf
  // LogNumbers: default: 16, stats: 10, pikachu: 5
@ -629,7 +629,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
  ASSERT_EQ("v2", Get("bar2"));
  ASSERT_EQ("v2", Get("foo2"));

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  // writing to default and stats cf, flushing default cf
  // LogNumbers: default: 19, stats: 19, pikachu: 19
@ -644,7 +644,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
  ASSERT_OK(Put(1, "Jolteon", "v3"));
  ASSERT_EQ("v3", Get(1, "Jolteon"));

-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  // writing to all three cf, flushing test cf
  // LogNumbers: default: 19, stats: 19, pikachu: 22