From 6727259eb46b0899d329965d19a99326191d4911 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Tue, 22 Sep 2020 09:55:42 -0700 Subject: [PATCH] Possible fix to flaky db_write_test (#7418) Summary: Make the test robust to spurious wakeups on condition variable, and clear sync points to ensure no use-after-free. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7418 Test Plan: repeated runs on updated test, watch CircleCI for recurrence Reviewed By: jay-zhuang Differential Revision: D23828823 Pulled By: pdillinger fbshipit-source-id: af85117d9c02602541a90252840e0e5a6996de5b --- db/db_write_test.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/db/db_write_test.cc b/db/db_write_test.cc index 97dc5bfc4..095292790 100644 --- a/db/db_write_test.cc +++ b/db/db_write_test.cc @@ -49,6 +49,8 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) { std::atomic thread_num(0); port::Mutex mutex; port::CondVar cv(&mutex); + // Guarded by mutex + int writers = 0; Reopen(options); @@ -68,6 +70,7 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) { }; std::function unblock_main_thread_func = [&](void *) { mutex.Lock(); + ++writers; cv.SignalAll(); mutex.Unlock(); }; @@ -106,18 +109,18 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) { mutex.Lock(); // First leader threads.emplace_back(write_slowdown_func); - cv.Wait(); + while (writers != 1) { + cv.Wait(); + } // Second leader. Will stall writes threads.emplace_back(write_slowdown_func); - cv.Wait(); threads.emplace_back(write_no_slowdown_func); - cv.Wait(); threads.emplace_back(write_slowdown_func); - cv.Wait(); threads.emplace_back(write_no_slowdown_func); - cv.Wait(); threads.emplace_back(write_slowdown_func); - cv.Wait(); + while (writers != 6) { + cv.Wait(); + } mutex.Unlock(); TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:1"); @@ -131,6 +134,8 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) { for (auto& t : threads) { t.join(); } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); } TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {