From 618d07b06860ba9d776d2bc768f4c33f08f18526 Mon Sep 17 00:00:00 2001 From: Venkatesh Radhakrishnan Date: Thu, 23 Apr 2015 08:35:02 -0700 Subject: [PATCH] Making PreShutdown tests more reliable. Summary: A couple of times on Travis, we have had the thread status say that there were no compactions done and since we assert for it, the test failed. We now fix this by waiting till compaction started. Test Plan: run DBTEST::*PreShutdown* d=/tmp/j; rm -rf $d; seq 200 | parallel --gnu --eta 'd=/tmp/j/d-{}; mkdir -p $d; TEST_TMPDIR=$d ./db_test --gtest_filter=DBTest.PreShutdown* >& '$d'/log-{}' Reviewers: sdong, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37545 --- db/db_test.cc | 52 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/db/db_test.cc b/db/db_test.cc index a1473d954..df631779c 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -10675,7 +10675,7 @@ TEST_F(DBTest, PreShutdownMultipleCompaction) { const int kTestKeySize = 16; const int kTestValueSize = 984; const int kEntrySize = kTestKeySize + kTestValueSize; - const int kEntriesPerBuffer = 10; + const int kEntriesPerBuffer = 40; const int kNumL0Files = 4; const int kHighPriCount = 3; @@ -10709,6 +10709,8 @@ TEST_F(DBTest, PreShutdownMultipleCompaction) { {{"FlushJob::FlushJob()", "CompactionJob::Run():Start"}, {"CompactionJob::Run():Start", "DBTest::PreShutdownMultipleCompaction:Preshutdown"}, + {"CompactionJob::Run():Start", + "DBTest::PreShutdownMultipleCompaction:VerifyCompaction"}, {"DBTest::PreShutdownMultipleCompaction:Preshutdown", "CompactionJob::Run():End"}, {"CompactionJob::Run():End", @@ -10718,10 +10720,9 @@ TEST_F(DBTest, PreShutdownMultipleCompaction) { // Make rocksdb busy int key = 0; - int max_operation_count[ThreadStatus::NUM_OP_TYPES] = {0}; // check how many threads are doing compaction using GetThreadList int operation_count[ThreadStatus::NUM_OP_TYPES] = {0}; - for (int file = 0; file < 8 * kNumL0Files; ++file) { + for (int file = 0; file < 16 * kNumL0Files; ++file) { for (int k = 0; k < kEntriesPerBuffer; ++k) { ASSERT_OK(Put(ToString(key++), RandomString(&rnd, kTestValueSize))); } @@ -10731,22 +10732,19 @@ TEST_F(DBTest, PreShutdownMultipleCompaction) { operation_count[thread.operation_type]++; } - // Record the max number of compactions at a time. - for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) { - if (max_operation_count[i] < operation_count[i]) { - max_operation_count[i] = operation_count[i]; - } - } // Speed up the test - if (max_operation_count[ThreadStatus::OP_FLUSH] > 1 && - max_operation_count[ThreadStatus::OP_COMPACTION] > + if (operation_count[ThreadStatus::OP_FLUSH] > 1 && + operation_count[ThreadStatus::OP_COMPACTION] > 0.6 * options.max_background_compactions) { break; } + if (file == 15 * kNumL0Files) { + TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:Preshutdown"); + } } TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:Preshutdown"); - ASSERT_GE(max_operation_count[ThreadStatus::OP_COMPACTION], 1); + ASSERT_GE(operation_count[ThreadStatus::OP_COMPACTION], 1); CancelAllBackgroundWork(db_); TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown"); dbfull()->TEST_WaitForCompact(); @@ -10765,7 +10763,7 @@ TEST_F(DBTest, PreShutdownCompactionMiddle) { const int kTestKeySize = 16; const int kTestValueSize = 984; const int kEntrySize = kTestKeySize + kTestValueSize; - const int kEntriesPerBuffer = 10; + const int kEntriesPerBuffer = 40; const int kNumL0Files = 4; const int kHighPriCount = 3; @@ -10796,20 +10794,21 @@ TEST_F(DBTest, PreShutdownCompactionMiddle) { std::vector thread_list; // Delay both flush and compaction rocksdb::SyncPoint::GetInstance()->LoadDependency( - {{"DBTest::PreShutdownMultipleCompaction:Preshutdown", + {{"DBTest::PreShutdownCompactionMiddle:Preshutdown", "CompactionJob::Run():Inprogress"}, + {"CompactionJob::Run():Start", + "DBTest::PreShutdownCompactionMiddle:VerifyCompaction"}, {"CompactionJob::Run():Inprogress", "CompactionJob::Run():End"}, {"CompactionJob::Run():End", - "DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown"}}); + "DBTest::PreShutdownCompactionMiddle:VerifyPreshutdown"}}); rocksdb::SyncPoint::GetInstance()->EnableProcessing(); // Make rocksdb busy int key = 0; - int max_operation_count[ThreadStatus::NUM_OP_TYPES] = {0}; // check how many threads are doing compaction using GetThreadList int operation_count[ThreadStatus::NUM_OP_TYPES] = {0}; - for (int file = 0; file < 8 * kNumL0Files; ++file) { + for (int file = 0; file < 16 * kNumL0Files; ++file) { for (int k = 0; k < kEntriesPerBuffer; ++k) { ASSERT_OK(Put(ToString(key++), RandomString(&rnd, kTestValueSize))); } @@ -10819,24 +10818,21 @@ TEST_F(DBTest, PreShutdownCompactionMiddle) { operation_count[thread.operation_type]++; } - // Record the max number of compactions at a time. - for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) { - if (max_operation_count[i] < operation_count[i]) { - max_operation_count[i] = operation_count[i]; - } - } // Speed up the test - if (max_operation_count[ThreadStatus::OP_FLUSH] > 1 && - max_operation_count[ThreadStatus::OP_COMPACTION] > + if (operation_count[ThreadStatus::OP_FLUSH] > 1 && + operation_count[ThreadStatus::OP_COMPACTION] > 0.6 * options.max_background_compactions) { break; } + if (file == 15 * kNumL0Files) { + TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:VerifyCompaction"); + } } - ASSERT_GE(max_operation_count[ThreadStatus::OP_COMPACTION], 1); + ASSERT_GE(operation_count[ThreadStatus::OP_COMPACTION], 1); CancelAllBackgroundWork(db_); - TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:Preshutdown"); - TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown"); + TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:Preshutdown"); + TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:VerifyPreshutdown"); dbfull()->TEST_WaitForCompact(); // Record the number of compactions at a time. for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) {