NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix FIFO Compaction with TTL tests

Browse Source 
Summary:
- FIFOCompactionWithTTLTest was flaky when run in parallel earlier, and hence it was disabled. Fixed it now.
- Also, faking sleep now instead of really sleeping to make tests more realistic by using TTLs like 1 hour and 1 day.
Closes https://github.com/facebook/rocksdb/pull/2650

Differential Revision: D5506038

Pulled By: sagar0

fbshipit-source-id: deb429a527f045e3e2c5138b547c3e8ac8586aa2
main
Sagar Vemuri 7 years ago committed by Facebook Github Bot
parent aaf42fe775
commit ac748c57ed
1 changed files with 35 additions and 25 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 60
      db/db_test.cc

60
db/db_test.cc
Unescape View File

@ -2847,20 +2847,23 @@ TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) {
ASSERT_TRUE(TryReopen(options).IsNotSupported());
}
TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
TEST_F(DBTest, FIFOCompactionWithTTLTest) {
Options options;
options.compaction_style = kCompactionStyleFIFO;
options.write_buffer_size = 10 << 10; // 10KB
options.arena_block_size = 4096;
options.compression = kNoCompression;
options.create_if_missing = true;
env_->time_elapse_only_sleep_ = false;
options.env = env_;
// Test to make sure that all files with expired ttl are deleted on next
// manual compaction.
{
env_->addon_time_.store(0);
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 600; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60 ; // 1 hour
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2871,19 +2874,21 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
// sleep for 5 seconds
env_->SleepForMicroseconds(5 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
// Note: Couldn't use SleepForMicroseconds because it takes an int instead
// of uint64_t. Hence used addon_time_ directly.
// env_->SleepForMicroseconds(2 * 60 * 60 * 1000 * 1000);
env_->addon_time_.fetch_add(2 * 60 * 60);
// Since no flushes and compactions have run, the db should still be in
// the same state even after considerable time has passed.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
// change ttl to 1 sec. So all files should be deleted on next compaction.
options.compaction_options_fifo.ttl = 1;
Reopen(options);
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
}
@ -2893,7 +2898,7 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
{
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 5; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2904,11 +2909,13 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
env_->SleepForMicroseconds(6 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
env_->addon_time_.fetch_add(2 * 60 * 60);
// Just to make sure that we are in the same state even after sleeping.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
@ -2930,10 +2937,10 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
// Test that shows the fall back to size-based FIFO compaction if TTL-based
// deletion doesn't move the total size to be less than max_table_files_size.
{
options.write_buffer_size = 110 << 10; // 10KB
options.write_buffer_size = 10 << 10; // 10KB
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 5; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2944,11 +2951,13 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 3);
env_->SleepForMicroseconds(6 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
env_->addon_time_.fetch_add(2 * 60 * 60);
// Just to make sure that we are in the same state even after sleeping.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 3);
@ -2957,8 +2966,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Size limit is still guaranteed.
ASSERT_LE(SizeAtLevel(0),
options.compaction_options_fifo.max_table_files_size);
@ -2968,7 +2977,7 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
{
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = true;
options.compaction_options_fifo.ttl = 5; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options.level0_file_num_compaction_trigger = 6;
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2980,15 +2989,16 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
// With Intra-L0 compaction, out of 10 files, 6 files will be compacted to 1
// (due to level0_file_num_compaction_trigger = 6).
// So total files = 1 + remaining 4 = 5.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
// Sleep for a little over ttl time.
env_->SleepForMicroseconds(6 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
env_->addon_time_.fetch_add(2 * 60 * 60);
// Just to make sure that we are in the same state even after sleeping.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
@ -2998,8 +3008,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
ASSERT_LE(SizeAtLevel(0),
options.compaction_options_fifo.max_table_files_size);
@ -3011,7 +3021,7 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
options.write_buffer_size = 20 << 10; // 20K
options.compaction_options_fifo.max_table_files_size = 1500 << 10; // 1.5MB
options.compaction_options_fifo.allow_compaction = true;
options.compaction_options_fifo.ttl = 60 * 60; // 1 hour
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options.level0_file_num_compaction_trigger = 6;
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -3023,8 +3033,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// It should be compacted to 10 files.
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
@ -3034,8 +3044,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j + 2000), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// It should be compacted to no more than 20 files.
ASSERT_GT(NumTableFilesAtLevel(0), 10);

Write Preview
Loading…
Cancel
Save