From 1078d860a9a08f915a11e7ea3ebf66c46e829b57 Mon Sep 17 00:00:00 2001 From: Jay Zhuang Date: Mon, 12 Dec 2022 10:37:55 -0800 Subject: [PATCH] Add an unittest for Periodic compaction conflict with ongoing compaction (#10908) Summary: Add a tiered storage migration test which would conflict with an ongoing penultimate level compaction. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10908 Test Plan: Test only change Reviewed By: anand1976 Differential Revision: D40864509 Pulled By: ajkr fbshipit-source-id: e316e849a01a6c71a41be130101f909b6c0498cb --- db/compaction/tiered_compaction_test.cc | 167 +++++++++++++++++++++--- db/db_impl/db_impl.h | 2 +- db/db_impl/db_impl_debug.cc | 2 +- db/periodic_task_scheduler_test.cc | 20 +-- db/seqno_time_test.cc | 52 ++++---- monitoring/stats_history_test.cc | 52 ++++---- 6 files changed, 211 insertions(+), 84 deletions(-) diff --git a/db/compaction/tiered_compaction_test.cc b/db/compaction/tiered_compaction_test.cc index fa603a3a5..f4837dcf9 100644 --- a/db/compaction/tiered_compaction_test.cc +++ b/db/compaction/tiered_compaction_test.cc @@ -1251,7 +1251,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); int sst_num = 0; @@ -1259,7 +1259,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) { for (; sst_num < kNumTrigger; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -1313,7 +1313,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); int sst_num = 0; @@ -1321,7 +1321,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) { for (; sst_num < kNumTrigger; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -1355,7 +1355,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) { for (int i = 0; i < kNumKeys; i++) { // the value needs to be big enough to trigger full compaction ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100))); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -1389,7 +1389,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); int sst_num = 0; @@ -1397,7 +1397,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) { for (; sst_num < kNumTrigger; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -1463,13 +1463,13 @@ TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) { Random rnd(301); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(rnd.Uniform(10) + 1)); }); for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(i), rnd.RandomString(100))); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(rnd.Uniform(2))); }); } @@ -1516,7 +1516,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); int sst_num = 0; @@ -1524,7 +1524,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) { for (; sst_num < kNumTrigger; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -1594,7 +1594,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); Random rnd(301); @@ -1603,7 +1603,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) { for (; sst_num < kNumTrigger; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100))); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -1696,7 +1696,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) { for (int i = 0; i < kNumKeys; i++) { // the value needs to be big enough to trigger full compaction ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -1721,6 +1721,133 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) { Close(); } +TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) { + // Test the last level only periodic compaction should also be blocked by an + // ongoing compaction in penultimate level if tiered compaction is enabled + // otherwise, the periodic compaction should just run for the last level. + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kPenultimateLevel = kNumLevels - 2; + const int kKeyPerSec = 1; + const int kNumKeys = 100; + + bool enable_preclude_last_level = GetParam(); + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preserve_internal_time_seconds = 20000; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + options.ignore_max_compaction_bytes_for_input = false; + options.periodic_compaction_seconds = 10000; + DestroyAndReopen(options); + + Random rnd(301); + + for (int i = 0; i < 3 * kNumKeys; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); + } + ASSERT_OK(Flush()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // make sure all data is compacted to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + // enable preclude feature + if (enable_preclude_last_level) { + options.preclude_last_level_data_seconds = 20000; + } + options.max_background_jobs = 8; + options.last_level_temperature = Temperature::kCold; + Reopen(options); + + std::atomic_bool is_size_ratio_compaction_running = false; + std::atomic_bool verified_last_level_compaction = false; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) { + auto compaction = static_cast(arg); + if (compaction->output_level() == kPenultimateLevel) { + is_size_ratio_compaction_running = true; + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction1"); + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction2"); + is_size_ratio_compaction_running = false; + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + auto compaction = static_cast(arg); + + if (is_size_ratio_compaction_running) { + if (enable_preclude_last_level) { + ASSERT_TRUE(compaction == nullptr); + } else { + ASSERT_TRUE(compaction != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kPeriodicCompaction); + ASSERT_EQ(compaction->start_level(), kNumLevels - 1); + } + verified_last_level_compaction = true; + } + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "AutoCompactionPicked"); + }); + + SyncPoint::GetInstance()->LoadDependency({ + {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction1", + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite"}, + {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "AutoCompactionPicked", + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction2"}, + }); + + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + for (int i = 0; i < kNumTrigger - 1; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(i * (kNumKeys - 1) + i), rnd.RandomString(10))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); + } + ASSERT_OK(Flush()); + } + + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite"); + + // wait for periodic compaction time and flush to trigger the periodic + // compaction, which should be blocked by ongoing compaction in the + // penultimate level + mock_clock_->MockSleepForSeconds(10000); + for (int i = 0; i < 3 * kNumKeys; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(10))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->WaitForCompact(true)); + + stop_token.reset(); + + Close(); +} + INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTestWithParms, PrecludeLastLevelTestWithParms, testing::Bool()); @@ -1781,14 +1908,14 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); Random rnd(301); for (int i = 0; i < 300; i++) { ASSERT_OK(Put(Key(i), rnd.RandomString(100))); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); } ASSERT_OK(Flush()); @@ -1892,7 +2019,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kSecondsPerKey)); }); @@ -1917,13 +2044,13 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) { Random rnd(301); for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(i + 3), rnd.RandomString(kValueBytes))); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); } auto* snap1 = db_->GetSnapshot(); for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(i), rnd.RandomString(kValueBytes))); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); } auto* snap2 = db_->GetSnapshot(); @@ -1937,7 +2064,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) { Key(2 * kNumKeysPerFile - 1), Key(2 * kNumKeysPerFile + 1))); ASSERT_OK(Flush()); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); verify_db(); diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index 725e77c18..920c3d3f8 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -1161,7 +1161,7 @@ class DBImpl : public DB { int TEST_BGCompactionsAllowed() const; int TEST_BGFlushesAllowed() const; size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const; - void TEST_WaitForPeridicTaskRun(std::function callback) const; + void TEST_WaitForPeriodicTaskRun(std::function callback) const; SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const; size_t TEST_EstimateInMemoryStatsHistorySize() const; diff --git a/db/db_impl/db_impl_debug.cc b/db/db_impl/db_impl_debug.cc index 7054b0669..83c37dced 100644 --- a/db/db_impl/db_impl_debug.cc +++ b/db/db_impl/db_impl_debug.cc @@ -290,7 +290,7 @@ size_t DBImpl::TEST_GetWalPreallocateBlockSize( } #ifndef ROCKSDB_LITE -void DBImpl::TEST_WaitForPeridicTaskRun(std::function callback) const { +void DBImpl::TEST_WaitForPeriodicTaskRun(std::function callback) const { periodic_task_scheduler_.TEST_WaitForRun(callback); } diff --git a/db/periodic_task_scheduler_test.cc b/db/periodic_task_scheduler_test.cc index 4abea4d5e..73c13fa13 100644 --- a/db/periodic_task_scheduler_test.cc +++ b/db/periodic_task_scheduler_test.cc @@ -65,7 +65,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) { ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec); ASSERT_GT(kPeriodSec, 1u); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec) - 1); }); @@ -77,14 +77,14 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) { ASSERT_EQ(1, pst_st_counter); ASSERT_EQ(1, flush_info_log_counter); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); ASSERT_EQ(2, dump_st_counter); ASSERT_EQ(2, pst_st_counter); ASSERT_EQ(2, flush_info_log_counter); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); ASSERT_EQ(3, dump_st_counter); @@ -98,7 +98,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) { ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec); // Info log flush should still run. - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); ASSERT_EQ(3, dump_st_counter); ASSERT_EQ(3, pst_st_counter); @@ -113,7 +113,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) { ASSERT_EQ(2, scheduler.TEST_GetValidTaskNum()); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); ASSERT_EQ(4, dump_st_counter); ASSERT_EQ(3, pst_st_counter); @@ -154,19 +154,19 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) { ASSERT_EQ(kInstanceNum * 3, scheduler.TEST_GetValidTaskNum()); int expected_run = kInstanceNum; - dbi->TEST_WaitForPeridicTaskRun( + dbi->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, pst_st_counter); expected_run += kInstanceNum; - dbi->TEST_WaitForPeridicTaskRun( + dbi->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, pst_st_counter); expected_run += kInstanceNum; - dbi->TEST_WaitForPeridicTaskRun( + dbi->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, pst_st_counter); @@ -178,9 +178,9 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) { expected_run += (kInstanceNum - half) * 2; - dbi->TEST_WaitForPeridicTaskRun( + dbi->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); - dbi->TEST_WaitForPeridicTaskRun( + dbi->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); ASSERT_EQ(expected_run, dump_st_counter); ASSERT_EQ(expected_run, pst_st_counter); diff --git a/db/seqno_time_test.cc b/db/seqno_time_test.cc index 12394a368..c19dd1c91 100644 --- a/db/seqno_time_test.cc +++ b/db/seqno_time_test.cc @@ -80,7 +80,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); int sst_num = 0; @@ -88,7 +88,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { for (; sst_num < kNumTrigger; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -110,7 +110,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { for (; sst_num < kNumTrigger * 2; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -124,7 +124,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { for (; sst_num < kNumTrigger * 3; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); } @@ -143,7 +143,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { AssertKeyTemperature(20, Temperature::kCold); for (int i = 0; i < 30; i++) { - dbfull()->TEST_WaitForPeridicTaskRun([&] { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { mock_clock_->MockSleepForSeconds(static_cast(20 * kKeyPerSec)); }); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); @@ -161,7 +161,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { // the compaction will not get the new seqno->time sampling to decide the last // a few data's time. for (int i = 0; i < 5; i++) { - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(1000)); }); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); } @@ -192,7 +192,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) { // pass some time first, otherwise the first a few keys write time are going // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); int sst_num = 0; @@ -200,7 +200,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) { for (; sst_num < 4; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); } ASSERT_OK(Flush()); @@ -222,7 +222,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) { for (; sst_num < 14; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); } ASSERT_OK(Flush()); @@ -243,7 +243,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) { // Wait some time, with each wait, the cold data is increasing and hot data is // decreasing for (int i = 0; i < 30; i++) { - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(200)); }); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); uint64_t pre_hot = hot_data_size; @@ -262,7 +262,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) { // hot data might not be empty, because if we don't write new data, there's // no seqno->time sampling available to the compaction for (int i = 0; i < 5; i++) { - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(1000)); }); ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); } @@ -324,7 +324,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { // Write a key every 10 seconds for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); } ASSERT_OK(Flush()); @@ -357,7 +357,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { // Write a key every 1 seconds for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i + 190), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(1)); }); } seq_end = dbfull()->GetLatestSequenceNumber(); @@ -393,7 +393,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { // Write a key every 200 seconds for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i + 380), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(200)); }); } seq_end = dbfull()->GetLatestSequenceNumber(); @@ -435,7 +435,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { // Write a key every 100 seconds for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i + 570), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } seq_end = dbfull()->GetLatestSequenceNumber(); @@ -516,7 +516,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { // Write some data and increase the current time for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } ASSERT_OK(Flush()); @@ -536,7 +536,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { // Write some data to the default CF (without preclude_last_level feature) for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } ASSERT_OK(Flush()); @@ -544,7 +544,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { // Write some data to the CF one for (int i = 0; i < 20; i++) { ASSERT_OK(Put(1, Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); } ASSERT_OK(Flush(1)); @@ -568,7 +568,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { // Add more data to CF "two" to fill the in memory mapping for (int i = 0; i < 2000; i++) { ASSERT_OK(Put(2, Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping(); @@ -592,7 +592,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { // enabled have flushed, the in-memory seqno->time mapping should be cleared for (int i = 0; i < 10; i++) { ASSERT_OK(Put(0, Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping(); @@ -603,7 +603,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { for (int j = 0; j < 3; j++) { for (int i = 0; i < 200; i++) { ASSERT_OK(Put(2, Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } ASSERT_OK(Flush(2)); @@ -623,7 +623,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { for (int j = 0; j < 2; j++) { for (int i = 0; i < 200; i++) { ASSERT_OK(Put(0, Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } ASSERT_OK(Flush(0)); @@ -638,7 +638,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { // Write some data to CF "two", but don't flush to accumulate for (int i = 0; i < 1000; i++) { ASSERT_OK(Put(2, Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } ASSERT_GE( @@ -682,7 +682,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiInstancesBasic) { WriteOptions wo; for (int i = 0; i < 200; i++) { ASSERT_OK(dbi->Put(wo, Key(i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } SeqnoToTimeMapping seqno_to_time_mapping = dbi->TEST_GetSeqnoToTimeMapping(); @@ -720,7 +720,7 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) { for (; sst_num < kNumTrigger - 1; sst_num++) { for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); } ASSERT_OK(Flush()); @@ -742,7 +742,7 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) { // Trigger a compaction for (int i = 0; i < kNumKeys; i++) { ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); } sst_num++; diff --git a/monitoring/stats_history_test.cc b/monitoring/stats_history_test.cc index 21ac786b4..fed8535f4 100644 --- a/monitoring/stats_history_test.cc +++ b/monitoring/stats_history_test.cc @@ -66,10 +66,10 @@ TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) { // Wait for the first stats persist to finish, as the initial delay could be // different. - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); ASSERT_GE(counter, 1); @@ -98,17 +98,17 @@ TEST_F(StatsHistoryTest, StatsPersistScheduling) { // Wait for the first stats persist to finish, as the initial delay could be // different. - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); ASSERT_GE(counter, 1); // Test cancel job through SetOptions ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}})); int old_val = counter; - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec * 2); }); ASSERT_EQ(counter, old_val); @@ -130,7 +130,7 @@ TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) { {{"stats_persist_period_sec", std::to_string(kPeriodSec)}})); ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); ASSERT_GE(counter, 1); Close(); @@ -149,11 +149,11 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { ReopenWithColumnFamilies({"default", "pikachu"}, options); // make sure the first stats persist to finish - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); // Wait for stats persist to finish - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); std::unique_ptr stats_iter; @@ -171,7 +171,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { ASSERT_GT(stats_count, 0); // Wait a bit and verify no more stats are found for (int i = 0; i < 10; ++i) { - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(1); }); } ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter)); @@ -226,7 +226,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { const int kIterations = 10; for (int i = 0; i < kIterations; ++i) { - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); } @@ -250,7 +250,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { // Wait for stats persist to finish for (int i = 0; i < kIterations; ++i) { - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); } @@ -299,11 +299,11 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { // Wait for the first stats persist to finish, as the initial delay could be // different. - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); // Wait for stats persist to finish - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); auto iter = @@ -311,14 +311,14 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { int key_count1 = countkeys(iter); delete iter; - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); int key_count2 = countkeys(iter); delete iter; - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); @@ -392,32 +392,32 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { // Wait for the first stats persist to finish, as the initial delay could be // different. - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); // Wait for stats persist to finish - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); auto iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); countkeys(iter); delete iter; - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); countkeys(iter); delete iter; - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); countkeys(iter); delete iter; - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); std::map stats_map_after; @@ -481,10 +481,10 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { ASSERT_EQ(Get(2, "foo"), "bar"); // make sure the first stats persist to finish - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); auto iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); @@ -581,7 +581,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) { // Wait for the first stats persist to finish, as the initial delay could be // different. - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); ColumnFamilyData* cfd_default = @@ -600,7 +600,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) { ASSERT_OK(Put(1, "Eevee", "v0")); ASSERT_EQ("v0", Get(1, "Eevee")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); // writing to all three cf, flush default cf // LogNumbers: default: 16, stats: 10, pikachu: 5 @@ -629,7 +629,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) { ASSERT_EQ("v2", Get("bar2")); ASSERT_EQ("v2", Get("foo2")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); // writing to default and stats cf, flushing default cf // LogNumbers: default: 19, stats: 19, pikachu: 19 @@ -644,7 +644,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) { ASSERT_OK(Put(1, "Jolteon", "v3")); ASSERT_EQ("v3", Get(1, "Jolteon")); - dbfull()->TEST_WaitForPeridicTaskRun( + dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); // writing to all three cf, flushing test cf // LogNumbers: default: 19, stats: 19, pikachu: 22