|
|
@ -126,11 +126,11 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) { |
|
|
|
for (int num = 0; num < 16; num++) { |
|
|
|
for (int num = 0; num < 16; num++) { |
|
|
|
// Write 100KB file. And immediately it should be compacted to one file.
|
|
|
|
// Write 100KB file. And immediately it should be compacted to one file.
|
|
|
|
GenerateNewFile(&rnd, &key_idx); |
|
|
|
GenerateNewFile(&rnd, &key_idx); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(NumSortedRuns(0), 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(0), 1); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_OK(Put(Key(key_idx), "")); |
|
|
|
ASSERT_OK(Put(Key(key_idx), "")); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(NumSortedRuns(0), 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(0), 1); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -162,15 +162,15 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) { |
|
|
|
Env::Priority::LOW); |
|
|
|
Env::Priority::LOW); |
|
|
|
|
|
|
|
|
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { |
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { |
|
|
|
Put(Key(num * 10), "val"); |
|
|
|
ASSERT_OK(Put(Key(num * 10), "val")); |
|
|
|
if (num) { |
|
|
|
if (num) { |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
} |
|
|
|
} |
|
|
|
Put(Key(30 + num * 10), "val"); |
|
|
|
ASSERT_OK(Put(Key(30 + num * 10), "val")); |
|
|
|
Put(Key(60 + num * 10), "val"); |
|
|
|
ASSERT_OK(Put(Key(60 + num * 10), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Put("", ""); |
|
|
|
ASSERT_OK(Put("", "")); |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
|
|
|
|
|
|
|
|
// Query set of non existing keys
|
|
|
|
// Query set of non existing keys
|
|
|
|
for (int i = 5; i < 90; i += 10) { |
|
|
|
for (int i = 5; i < 90; i += 10) { |
|
|
@ -190,7 +190,7 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) { |
|
|
|
|
|
|
|
|
|
|
|
// Unblock compaction and wait it for happening.
|
|
|
|
// Unblock compaction and wait it for happening.
|
|
|
|
sleeping_task_low.WakeUp(); |
|
|
|
sleeping_task_low.WakeUp(); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
// The same queries will not trigger bloom filter
|
|
|
|
// The same queries will not trigger bloom filter
|
|
|
|
for (int i = 5; i < 90; i += 10) { |
|
|
|
for (int i = 5; i < 90; i += 10) { |
|
|
@ -294,7 +294,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) { |
|
|
|
// Now we have 3 files at level 0, with size 4, 2.4, 2. Let's generate a
|
|
|
|
// Now we have 3 files at level 0, with size 4, 2.4, 2. Let's generate a
|
|
|
|
// new file of size 1.
|
|
|
|
// new file of size 1.
|
|
|
|
GenerateNewFile(1, &rnd, &key_idx); |
|
|
|
GenerateNewFile(1, &rnd, &key_idx); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Level-0 compaction is triggered, but no file will be picked up.
|
|
|
|
// Level-0 compaction is triggered, but no file will be picked up.
|
|
|
|
ASSERT_EQ(NumSortedRuns(1), 4); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 4); |
|
|
|
|
|
|
|
|
|
|
@ -303,7 +303,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) { |
|
|
|
// a new file of size 1.
|
|
|
|
// a new file of size 1.
|
|
|
|
filter->expect_full_compaction_.store(true); |
|
|
|
filter->expect_full_compaction_.store(true); |
|
|
|
GenerateNewFile(1, &rnd, &key_idx); |
|
|
|
GenerateNewFile(1, &rnd, &key_idx); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// All files at level 0 will be compacted into a single one.
|
|
|
|
// All files at level 0 will be compacted into a single one.
|
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
|
|
|
|
|
|
|
|
@ -336,7 +336,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) { |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 2); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 2); |
|
|
@ -346,7 +346,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) { |
|
|
|
// but will instead trigger size amplification.
|
|
|
|
// but will instead trigger size amplification.
|
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
// Verify that size amplification did occur
|
|
|
|
// Verify that size amplification did occur
|
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
@ -394,7 +394,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) { |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 2); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 2); |
|
|
@ -404,7 +404,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) { |
|
|
|
// but could instead trigger size amplification if it's set
|
|
|
|
// but could instead trigger size amplification if it's set
|
|
|
|
// to 110.
|
|
|
|
// to 110.
|
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Verify compaction did not happen
|
|
|
|
// Verify compaction did not happen
|
|
|
|
ASSERT_EQ(NumSortedRuns(1), 3); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 3); |
|
|
|
|
|
|
|
|
|
|
@ -425,7 +425,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) { |
|
|
|
ASSERT_EQ(110u, mutable_cf_options.compaction_options_universal |
|
|
|
ASSERT_EQ(110u, mutable_cf_options.compaction_options_universal |
|
|
|
.max_size_amplification_percent); |
|
|
|
.max_size_amplification_percent); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Verify that size amplification did happen
|
|
|
|
// Verify that size amplification did happen
|
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
|
ASSERT_EQ(total_picked_compactions, 1); |
|
|
|
ASSERT_EQ(total_picked_compactions, 1); |
|
|
@ -473,7 +473,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionReadAmplification) { |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger); |
|
|
@ -481,7 +481,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionReadAmplification) { |
|
|
|
// Flush whatever is remaining in memtable. This is typically small, about
|
|
|
|
// Flush whatever is remaining in memtable. This is typically small, about
|
|
|
|
// 30KB.
|
|
|
|
// 30KB.
|
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Verify compaction did not happen
|
|
|
|
// Verify compaction did not happen
|
|
|
|
ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger + 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger + 1); |
|
|
|
ASSERT_EQ(total_picked_compactions, 0); |
|
|
|
ASSERT_EQ(total_picked_compactions, 0); |
|
|
@ -510,7 +510,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionReadAmplification) { |
|
|
|
ASSERT_EQ(mutable_cf_options.compaction_options_universal.max_merge_width, |
|
|
|
ASSERT_EQ(mutable_cf_options.compaction_options_universal.max_merge_width, |
|
|
|
2u); |
|
|
|
2u); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
// Files in L0 are approx: 0.3 (30KB), 1, 1, 1.
|
|
|
|
// Files in L0 are approx: 0.3 (30KB), 1, 1, 1.
|
|
|
|
// On compaction: the files are below the size amp threshold, so we
|
|
|
|
// On compaction: the files are below the size amp threshold, so we
|
|
|
@ -550,8 +550,8 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) { |
|
|
|
for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) { |
|
|
|
for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) { |
|
|
|
ASSERT_OK(Put(1, ToString(key), rnd.RandomString(kTestValueSize))); |
|
|
|
ASSERT_OK(Put(1, ToString(key), rnd.RandomString(kTestValueSize))); |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ColumnFamilyMetaData cf_meta; |
|
|
|
ColumnFamilyMetaData cf_meta; |
|
|
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); |
|
|
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); |
|
|
|
std::vector<std::string> compaction_input_file_names; |
|
|
|
std::vector<std::string> compaction_input_file_names; |
|
|
@ -631,7 +631,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) { |
|
|
|
compact_options.change_level = true; |
|
|
|
compact_options.change_level = true; |
|
|
|
compact_options.target_level = 4; |
|
|
|
compact_options.target_level = 4; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
db_->CompactRange(compact_options, nullptr, nullptr); |
|
|
|
ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); |
|
|
|
ASSERT_EQ("0,0,0,0,1", FilesPerLevel(0)); |
|
|
|
ASSERT_EQ("0,0,0,0,1", FilesPerLevel(0)); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -665,7 +665,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) { |
|
|
|
ASSERT_OK(Put(1, Key(i % num_keys), Key(i))); |
|
|
|
ASSERT_OK(Put(1, Key(i % num_keys), Key(i))); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
for (int i = num_keys; i < num_keys * 2; i++) { |
|
|
|
for (int i = num_keys; i < num_keys * 2; i++) { |
|
|
|
ASSERT_EQ(Get(1, Key(i % num_keys)), Key(i)); |
|
|
|
ASSERT_EQ(Get(1, Key(i % num_keys)), Key(i)); |
|
|
@ -712,7 +712,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) { |
|
|
|
std::vector<std::string> values; |
|
|
|
std::vector<std::string> values; |
|
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
ASSERT_GT(trivial_move, 0); |
|
|
|
ASSERT_GT(trivial_move, 0); |
|
|
|
ASSERT_GT(non_trivial_move, 0); |
|
|
|
ASSERT_GT(non_trivial_move, 0); |
|
|
@ -775,7 +775,7 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) { |
|
|
|
for (int i = 0; i < num_keys * 2; i++) { |
|
|
|
for (int i = 0; i < num_keys * 2; i++) { |
|
|
|
ASSERT_OK(Put(1, Key(i % num_keys), Key(i))); |
|
|
|
ASSERT_OK(Put(1, Key(i % num_keys), Key(i))); |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
|
|
|
ASSERT_EQ(num_compactions_running.load(), 0); |
|
|
|
ASSERT_EQ(num_compactions_running.load(), 0); |
|
|
@ -853,7 +853,7 @@ TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) { |
|
|
|
|
|
|
|
|
|
|
|
// Hold the 1st compaction from finishing
|
|
|
|
// Hold the 1st compaction from finishing
|
|
|
|
TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2"); |
|
|
|
TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2"); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
// There should only be one picked compaction as the score drops below one
|
|
|
|
// There should only be one picked compaction as the score drops below one
|
|
|
|
// after the first one is picked.
|
|
|
|
// after the first one is picked.
|
|
|
@ -901,7 +901,7 @@ TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) { |
|
|
|
|
|
|
|
|
|
|
|
// Hold the 1st and 2nd compaction from finishing
|
|
|
|
// Hold the 1st and 2nd compaction from finishing
|
|
|
|
TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2"); |
|
|
|
TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2"); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
// This time we will trigger a compaction because of size ratio and
|
|
|
|
// This time we will trigger a compaction because of size ratio and
|
|
|
|
// another compaction because of number of files that are not compacted
|
|
|
|
// another compaction because of number of files that are not compacted
|
|
|
@ -935,14 +935,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) { |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(990))); |
|
|
|
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(990))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
|
|
|
|
|
|
|
|
if (num < options.level0_file_num_compaction_trigger - 1) { |
|
|
|
if (num < options.level0_file_num_compaction_trigger - 1) { |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), num + 1); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(1), 1); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -973,7 +973,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
ASSERT_EQ(NumSortedRuns(), num + 1); |
|
|
|
ASSERT_EQ(NumSortedRuns(), num + 1); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -983,7 +983,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Suppose each file flushed from mem table has size 1. Now we compact
|
|
|
|
// Suppose each file flushed from mem table has size 1. Now we compact
|
|
|
|
// (level0_file_num_compaction_trigger+1)=4 files and should have a big
|
|
|
|
// (level0_file_num_compaction_trigger+1)=4 files and should have a big
|
|
|
|
// file of size 4.
|
|
|
|
// file of size 4.
|
|
|
@ -1004,7 +1004,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
ASSERT_EQ(NumSortedRuns(), num + 3); |
|
|
|
ASSERT_EQ(NumSortedRuns(), num + 3); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -1014,7 +1014,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1.
|
|
|
|
// Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1.
|
|
|
|
// After compaction, we should have 3 files, with size 4, 0.4, 2.
|
|
|
|
// After compaction, we should have 3 files, with size 4, 0.4, 2.
|
|
|
|
ASSERT_EQ(NumSortedRuns(), 3); |
|
|
|
ASSERT_EQ(NumSortedRuns(), 3); |
|
|
@ -1025,7 +1025,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Level-0 compaction is triggered, but no file will be picked up.
|
|
|
|
// Level-0 compaction is triggered, but no file will be picked up.
|
|
|
|
ASSERT_EQ(NumSortedRuns(), 4); |
|
|
|
ASSERT_EQ(NumSortedRuns(), 4); |
|
|
|
} |
|
|
|
} |
|
|
@ -1054,8 +1054,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_LT(TotalSize(), 110000U * 2 * 0.9); |
|
|
|
ASSERT_LT(TotalSize(), 110000U * 2 * 0.9); |
|
|
|
|
|
|
|
|
|
|
@ -1066,8 +1066,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_LT(TotalSize(), 110000 * 4 * 0.9); |
|
|
|
ASSERT_LT(TotalSize(), 110000 * 4 * 0.9); |
|
|
|
|
|
|
|
|
|
|
@ -1079,8 +1079,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_LT(TotalSize(), 110000 * 6 * 0.9); |
|
|
|
ASSERT_LT(TotalSize(), 110000 * 6 * 0.9); |
|
|
|
|
|
|
|
|
|
|
@ -1092,8 +1092,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_GT(TotalSize(), 110000 * 11 * 0.8 + 110000 * 2); |
|
|
|
ASSERT_GT(TotalSize(), 110000 * 11 * 0.8 + 110000 * 2); |
|
|
|
} |
|
|
|
} |
|
|
@ -1122,8 +1122,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) { |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); |
|
|
|
key_idx++; |
|
|
|
key_idx++; |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_LT(TotalSize(), 120000U * 12 * 0.82 + 120000 * 2); |
|
|
|
ASSERT_LT(TotalSize(), 120000U * 12 * 0.82 + 120000 * 2); |
|
|
|
} |
|
|
|
} |
|
|
@ -1169,7 +1169,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) { |
|
|
|
std::vector<std::string> values; |
|
|
|
std::vector<std::string> values; |
|
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
ASSERT_GT(trivial_move, 0); |
|
|
|
ASSERT_GT(trivial_move, 0); |
|
|
|
ASSERT_GT(non_trivial_move, 0); |
|
|
|
ASSERT_GT(non_trivial_move, 0); |
|
|
@ -1215,7 +1215,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) { |
|
|
|
std::vector<std::string> values; |
|
|
|
std::vector<std::string> values; |
|
|
|
|
|
|
|
|
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
ASSERT_GT(trivial_move, 0); |
|
|
|
ASSERT_GT(trivial_move, 0); |
|
|
|
|
|
|
|
|
|
|
@ -1239,12 +1239,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) { |
|
|
|
options.num_levels = 1; |
|
|
|
options.num_levels = 1; |
|
|
|
|
|
|
|
|
|
|
|
std::vector<std::string> filenames; |
|
|
|
std::vector<std::string> filenames; |
|
|
|
env_->GetChildren(options.db_paths[1].path, &filenames); |
|
|
|
if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) { |
|
|
|
// Delete archival files.
|
|
|
|
// Delete archival files.
|
|
|
|
for (size_t i = 0; i < filenames.size(); ++i) { |
|
|
|
for (size_t i = 0; i < filenames.size(); ++i) { |
|
|
|
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i]); |
|
|
|
ASSERT_OK( |
|
|
|
|
|
|
|
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i])); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
ASSERT_OK(env_->DeleteDir(options.db_paths[1].path)); |
|
|
|
} |
|
|
|
} |
|
|
|
env_->DeleteDir(options.db_paths[1].path); |
|
|
|
|
|
|
|
Reopen(options); |
|
|
|
Reopen(options); |
|
|
|
|
|
|
|
|
|
|
|
Random rnd(301); |
|
|
|
Random rnd(301); |
|
|
@ -1503,11 +1505,11 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) { |
|
|
|
for (int i = 0; i <= max_key1; i++) { |
|
|
|
for (int i = 0; i <= max_key1; i++) { |
|
|
|
// each value is 10K
|
|
|
|
// each value is 10K
|
|
|
|
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); |
|
|
|
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
// Stage 2: reopen with universal compaction, num_levels=4
|
|
|
|
// Stage 2: reopen with universal compaction, num_levels=4
|
|
|
|
options.compaction_style = kCompactionStyleUniversal; |
|
|
|
options.compaction_style = kCompactionStyleUniversal; |
|
|
@ -1521,11 +1523,11 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) { |
|
|
|
for (int i = max_key1 + 1; i <= max_key2; i++) { |
|
|
|
for (int i = max_key1 + 1; i <= max_key2; i++) { |
|
|
|
// each value is 10K
|
|
|
|
// each value is 10K
|
|
|
|
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); |
|
|
|
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
verify_func(max_key2); |
|
|
|
verify_func(max_key2); |
|
|
|
// Compaction to non-L0 has happened.
|
|
|
|
// Compaction to non-L0 has happened.
|
|
|
@ -1540,7 +1542,8 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) { |
|
|
|
compact_options.change_level = true; |
|
|
|
compact_options.change_level = true; |
|
|
|
compact_options.target_level = 0; |
|
|
|
compact_options.target_level = 0; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
dbfull()->CompactRange(compact_options, handles_[1], nullptr, nullptr); |
|
|
|
ASSERT_OK( |
|
|
|
|
|
|
|
dbfull()->CompactRange(compact_options, handles_[1], nullptr, nullptr)); |
|
|
|
// Need to restart it once to remove higher level records in manifest.
|
|
|
|
// Need to restart it once to remove higher level records in manifest.
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, options); |
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, options); |
|
|
|
// Final reopen
|
|
|
|
// Final reopen
|
|
|
@ -1553,11 +1556,11 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) { |
|
|
|
for (int i = max_key2 + 1; i <= max_key3; i++) { |
|
|
|
for (int i = max_key2 + 1; i <= max_key3; i++) { |
|
|
|
// each value is 10K
|
|
|
|
// each value is 10K
|
|
|
|
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); |
|
|
|
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); |
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
} |
|
|
|
} |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
ASSERT_OK(Flush(1)); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
verify_func(max_key3); |
|
|
|
verify_func(max_key3); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -1579,12 +1582,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) { |
|
|
|
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); |
|
|
|
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); |
|
|
|
|
|
|
|
|
|
|
|
std::vector<std::string> filenames; |
|
|
|
std::vector<std::string> filenames; |
|
|
|
env_->GetChildren(options.db_paths[1].path, &filenames); |
|
|
|
if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) { |
|
|
|
// Delete archival files.
|
|
|
|
// Delete archival files.
|
|
|
|
for (size_t i = 0; i < filenames.size(); ++i) { |
|
|
|
for (size_t i = 0; i < filenames.size(); ++i) { |
|
|
|
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i]); |
|
|
|
ASSERT_OK( |
|
|
|
|
|
|
|
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i])); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
ASSERT_OK(env_->DeleteDir(options.db_paths[1].path)); |
|
|
|
} |
|
|
|
} |
|
|
|
env_->DeleteDir(options.db_paths[1].path); |
|
|
|
|
|
|
|
Reopen(options); |
|
|
|
Reopen(options); |
|
|
|
|
|
|
|
|
|
|
|
Random rnd(301); |
|
|
|
Random rnd(301); |
|
|
@ -1699,14 +1704,14 @@ TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) { |
|
|
|
// use no_wait above because that one waits for flush and compaction. We
|
|
|
|
// use no_wait above because that one waits for flush and compaction. We
|
|
|
|
// don't want to wait for compaction because the full compaction is
|
|
|
|
// don't want to wait for compaction because the full compaction is
|
|
|
|
// intentionally blocked while more files are flushed.
|
|
|
|
// intentionally blocked while more files are flushed.
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); |
|
|
|
} |
|
|
|
} |
|
|
|
if (i == 0) { |
|
|
|
if (i == 0) { |
|
|
|
TEST_SYNC_POINT( |
|
|
|
TEST_SYNC_POINT( |
|
|
|
"DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"); |
|
|
|
"DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
// First compaction should output to bottom level. Second should output to L0
|
|
|
|
// First compaction should output to bottom level. Second should output to L0
|
|
|
|
// since older L0 files pending compaction prevent it from being placed lower.
|
|
|
|
// since older L0 files pending compaction prevent it from being placed lower.
|
|
|
@ -1745,7 +1750,7 @@ TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) { |
|
|
|
int key_idx = 0; |
|
|
|
int key_idx = 0; |
|
|
|
GenerateNewFile(&rnd, &key_idx); |
|
|
|
GenerateNewFile(&rnd, &key_idx); |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
// Compacting the first four files was enough to bring the score below one so
|
|
|
|
// Compacting the first four files was enough to bring the score below one so
|
|
|
|
// there's no need to schedule any more compactions.
|
|
|
|
// there's no need to schedule any more compactions.
|
|
|
|
ASSERT_EQ(1, num_compactions_attempted); |
|
|
|
ASSERT_EQ(1, num_compactions_attempted); |
|
|
@ -1775,9 +1780,9 @@ TEST_P(DBTestUniversalCompaction, FinalSortedRunCompactFilesConflict) { |
|
|
|
auto stop_token = |
|
|
|
auto stop_token = |
|
|
|
dbfull()->TEST_write_controler().GetCompactionPressureToken(); |
|
|
|
dbfull()->TEST_write_controler().GetCompactionPressureToken(); |
|
|
|
|
|
|
|
|
|
|
|
Put("key", "val"); |
|
|
|
ASSERT_OK(Put("key", "val")); |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); |
|
|
|
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(num_levels_ - 1), 1); |
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(num_levels_ - 1), 1); |
|
|
|
ColumnFamilyMetaData cf_meta; |
|
|
|
ColumnFamilyMetaData cf_meta; |
|
|
|
ColumnFamilyHandle* default_cfh = db_->DefaultColumnFamily(); |
|
|
|
ColumnFamilyHandle* default_cfh = db_->DefaultColumnFamily(); |
|
|
@ -1801,10 +1806,10 @@ TEST_P(DBTestUniversalCompaction, FinalSortedRunCompactFilesConflict) { |
|
|
|
TEST_SYNC_POINT( |
|
|
|
TEST_SYNC_POINT( |
|
|
|
"DBTestUniversalCompaction:FinalSortedRunCompactFilesConflict:0"); |
|
|
|
"DBTestUniversalCompaction:FinalSortedRunCompactFilesConflict:0"); |
|
|
|
for (int i = 0; i < 2; ++i) { |
|
|
|
for (int i = 0; i < 2; ++i) { |
|
|
|
Put("key", "val"); |
|
|
|
ASSERT_OK(Put("key", "val")); |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
} |
|
|
|
} |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
compact_files_thread.join(); |
|
|
|
compact_files_thread.join(); |
|
|
|
} |
|
|
|
} |
|
|
@ -1835,7 +1840,7 @@ TEST_P(DBTestUniversalManualCompactionOutputPathId, |
|
|
|
DestroyAndReopen(options); |
|
|
|
DestroyAndReopen(options); |
|
|
|
CreateAndReopenWithCF({"pikachu"}, options); |
|
|
|
CreateAndReopenWithCF({"pikachu"}, options); |
|
|
|
MakeTables(3, "p", "q", 1); |
|
|
|
MakeTables(3, "p", "q", 1); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(2, TotalLiveFiles(1)); |
|
|
|
ASSERT_EQ(2, TotalLiveFiles(1)); |
|
|
|
ASSERT_EQ(2, GetSstFileCount(options.db_paths[0].path)); |
|
|
|
ASSERT_EQ(2, GetSstFileCount(options.db_paths[0].path)); |
|
|
|
ASSERT_EQ(0, GetSstFileCount(options.db_paths[1].path)); |
|
|
|
ASSERT_EQ(0, GetSstFileCount(options.db_paths[1].path)); |
|
|
@ -1844,7 +1849,7 @@ TEST_P(DBTestUniversalManualCompactionOutputPathId, |
|
|
|
CompactRangeOptions compact_options; |
|
|
|
CompactRangeOptions compact_options; |
|
|
|
compact_options.target_path_id = 1; |
|
|
|
compact_options.target_path_id = 1; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
db_->CompactRange(compact_options, handles_[1], nullptr, nullptr); |
|
|
|
ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); |
|
|
|
ASSERT_EQ(1, TotalLiveFiles(1)); |
|
|
|
ASSERT_EQ(1, TotalLiveFiles(1)); |
|
|
|
ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path)); |
|
|
|
ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path)); |
|
|
|
ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); |
|
|
|
ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); |
|
|
@ -1867,7 +1872,7 @@ TEST_P(DBTestUniversalManualCompactionOutputPathId, |
|
|
|
// Full compaction to DB path 0
|
|
|
|
// Full compaction to DB path 0
|
|
|
|
compact_options.target_path_id = 0; |
|
|
|
compact_options.target_path_id = 0; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; |
|
|
|
db_->CompactRange(compact_options, handles_[1], nullptr, nullptr); |
|
|
|
ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); |
|
|
|
ASSERT_EQ(1, TotalLiveFiles(1)); |
|
|
|
ASSERT_EQ(1, TotalLiveFiles(1)); |
|
|
|
ASSERT_EQ(1, GetSstFileCount(options.db_paths[0].path)); |
|
|
|
ASSERT_EQ(1, GetSstFileCount(options.db_paths[0].path)); |
|
|
|
ASSERT_EQ(0, GetSstFileCount(options.db_paths[1].path)); |
|
|
|
ASSERT_EQ(0, GetSstFileCount(options.db_paths[1].path)); |
|
|
@ -1904,23 +1909,23 @@ TEST_F(DBTestUniversalCompaction2, BasicL0toL1) { |
|
|
|
// during flush
|
|
|
|
// during flush
|
|
|
|
int i; |
|
|
|
int i; |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
// MoveFilesToLevel(6);
|
|
|
|
// MoveFilesToLevel(6);
|
|
|
|
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); |
|
|
|
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
|
|
|
|
|
|
|
|
|
|
|
for (i = 1999; i < kNumKeys; ++i) { |
|
|
|
for (i = 1999; i < kNumKeys; ++i) { |
|
|
|
if (i >= kNumKeys - kWindowSize && |
|
|
|
if (i >= kNumKeys - kWindowSize && |
|
|
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) { |
|
|
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) { |
|
|
|
Delete(Key(i)); |
|
|
|
ASSERT_OK(Delete(Key(i))); |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
} |
|
|
|
} |
|
|
@ -1947,21 +1952,21 @@ TEST_F(DBTestUniversalCompaction2, SingleLevel) { |
|
|
|
// during flush
|
|
|
|
// during flush
|
|
|
|
int i; |
|
|
|
int i; |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
for (i = 1999; i < kNumKeys; ++i) { |
|
|
|
for (i = 1999; i < kNumKeys; ++i) { |
|
|
|
if (i >= kNumKeys - kWindowSize && |
|
|
|
if (i >= kNumKeys - kWindowSize && |
|
|
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) { |
|
|
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) { |
|
|
|
Delete(Key(i)); |
|
|
|
ASSERT_OK(Delete(Key(i))); |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(1, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(1, NumTableFilesAtLevel(0)); |
|
|
|
} |
|
|
|
} |
|
|
|
#endif // ENABLE_SINGLE_LEVEL_DTC
|
|
|
|
#endif // ENABLE_SINGLE_LEVEL_DTC
|
|
|
@ -1985,50 +1990,50 @@ TEST_F(DBTestUniversalCompaction2, MultipleLevels) { |
|
|
|
// during flush
|
|
|
|
// during flush
|
|
|
|
int i; |
|
|
|
int i; |
|
|
|
for (i = 0; i < 500; ++i) { |
|
|
|
for (i = 0; i < 500; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 500; i < 1000; ++i) { |
|
|
|
for (i = 500; i < 1000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 1000; i < 1500; ++i) { |
|
|
|
for (i = 1000; i < 1500; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 1500; i < 2000; ++i) { |
|
|
|
for (i = 1500; i < 2000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
|
|
|
|
|
|
|
|
for (i = 1999; i < 2333; ++i) { |
|
|
|
for (i = 1999; i < 2333; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 2333; i < 2666; ++i) { |
|
|
|
for (i = 2333; i < 2666; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 2666; i < 2999; ++i) { |
|
|
|
for (i = 2666; i < 2999; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(5), 0); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(5), 0); |
|
|
|
|
|
|
|
|
|
|
|
for (i = 1900; i < 2100; ++i) { |
|
|
|
for (i = 1900; i < 2100; ++i) { |
|
|
|
Delete(Key(i)); |
|
|
|
ASSERT_OK(Delete(Key(i))); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(1)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(1)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(2)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(2)); |
|
|
@ -2057,23 +2062,23 @@ TEST_F(DBTestUniversalCompaction2, OverlappingL0) { |
|
|
|
// during flush
|
|
|
|
// during flush
|
|
|
|
int i; |
|
|
|
int i; |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 2000; i < 3000; ++i) { |
|
|
|
for (i = 2000; i < 3000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 3500; i < 4000; ++i) { |
|
|
|
for (i = 3500; i < 4000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
for (i = 2900; i < 3100; ++i) { |
|
|
|
for (i = 2900; i < 3100; ++i) { |
|
|
|
Delete(Key(i)); |
|
|
|
ASSERT_OK(Delete(Key(i))); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(2, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(2, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(6), 0); |
|
|
|
} |
|
|
|
} |
|
|
@ -2099,23 +2104,23 @@ TEST_F(DBTestUniversalCompaction2, IngestBehind) { |
|
|
|
// during flush
|
|
|
|
// during flush
|
|
|
|
int i; |
|
|
|
int i; |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
for (i = 0; i < 2000; ++i) { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
// MoveFilesToLevel(6);
|
|
|
|
// MoveFilesToLevel(6);
|
|
|
|
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); |
|
|
|
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
|
|
|
|
|
|
|
|
|
|
|
for (i = 1999; i < kNumKeys; ++i) { |
|
|
|
for (i = 1999; i < kNumKeys; ++i) { |
|
|
|
if (i >= kNumKeys - kWindowSize && |
|
|
|
if (i >= kNumKeys - kWindowSize && |
|
|
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) { |
|
|
|
i < kNumKeys - kWindowSize + kNumDelsTrigger) { |
|
|
|
Delete(Key(i)); |
|
|
|
ASSERT_OK(Delete(Key(i))); |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
Put(Key(i), "val"); |
|
|
|
ASSERT_OK(Put(Key(i), "val")); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(0)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(6)); |
|
|
|
ASSERT_EQ(0, NumTableFilesAtLevel(6)); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(5), 0); |
|
|
|
ASSERT_GT(NumTableFilesAtLevel(5), 0); |
|
|
@ -2188,7 +2193,7 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) { |
|
|
|
// Another flush would trigger compaction the oldest file.
|
|
|
|
// Another flush would trigger compaction the oldest file.
|
|
|
|
ASSERT_OK(Put("foo", "bar2")); |
|
|
|
ASSERT_OK(Put("foo", "bar2")); |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
|
|
|
|
|
|
|
|
ASSERT_EQ(1, periodic_compactions); |
|
|
|
ASSERT_EQ(1, periodic_compactions); |
|
|
|
ASSERT_EQ(0, start_level); |
|
|
|
ASSERT_EQ(0, start_level); |
|
|
@ -2199,7 +2204,7 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) { |
|
|
|
// A flush doesn't trigger a periodic compaction when threshold not hit
|
|
|
|
// A flush doesn't trigger a periodic compaction when threshold not hit
|
|
|
|
ASSERT_OK(Put("foo", "bar2")); |
|
|
|
ASSERT_OK(Put("foo", "bar2")); |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(0, periodic_compactions); |
|
|
|
ASSERT_EQ(0, periodic_compactions); |
|
|
|
|
|
|
|
|
|
|
|
// After periodic compaction threshold hits, a flush will trigger
|
|
|
|
// After periodic compaction threshold hits, a flush will trigger
|
|
|
@ -2207,7 +2212,7 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) { |
|
|
|
ASSERT_OK(Put("foo", "bar2")); |
|
|
|
ASSERT_OK(Put("foo", "bar2")); |
|
|
|
env_->MockSleepForSeconds(48 * 60 * 60 + 100); |
|
|
|
env_->MockSleepForSeconds(48 * 60 * 60 + 100); |
|
|
|
Flush(); |
|
|
|
Flush(); |
|
|
|
dbfull()->TEST_WaitForCompact(); |
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
|
|
|
ASSERT_EQ(1, periodic_compactions); |
|
|
|
ASSERT_EQ(1, periodic_compactions); |
|
|
|
ASSERT_EQ(0, start_level); |
|
|
|
ASSERT_EQ(0, start_level); |
|
|
|
ASSERT_EQ(4, output_level); |
|
|
|
ASSERT_EQ(4, output_level); |
|
|
|