From e1c7209bebfde34a03b397cf6cb6e02aeb864118 Mon Sep 17 00:00:00 2001 From: Changyu Bi Date: Tue, 30 May 2023 15:00:05 -0700 Subject: [PATCH] Fix flaky test: `DBCompactionTest.WaitForCompactShutdownWhileWaiting` (#11488) Summary: tsan complains with the following error message. This is likely due to DB object destroyed while WaitForCompact() is still running. ``` [ RUN ] DBCompactionTest.WaitForCompactShutdownWhileWaiting ================== WARNING: ThreadSanitizer: data race (pid=1128703) Atomic read of size 1 at 0x7b8c00000740 by thread T4: #0 pthread_cond_wait (db_compaction_test+0x46970a) https://github.com/facebook/rocksdb/issues/1 rocksdb::port::CondVar::Wait() /root/project/port/port_posix.cc:119:23 (librocksdb.so.8.4+0x7c4c60) https://github.com/facebook/rocksdb/issues/2 rocksdb::InstrumentedCondVar::WaitInternal() /root/project/monitoring/instrumented_mutex.cc:69:9 (librocksdb.so.8.4+0x75f697) https://github.com/facebook/rocksdb/issues/3 rocksdb::InstrumentedCondVar::Wait() /root/project/monitoring/instrumented_mutex.cc:62:3 (librocksdb.so.8.4+0x75f697) https://github.com/facebook/rocksdb/issues/4 rocksdb::DBImpl::WaitForCompact(rocksdb::WaitForCompactOptions const&) /root/project/db/db_impl/db_impl_compaction_flush.cc:3978:14 (librocksdb.so.8.4+0x494174) https://github.com/facebook/rocksdb/issues/5 rocksdb::DBCompactionTest_WaitForCompactShutdownWhileWaiting_Test::TestBody()::$_30::operator()() const /root/project/db/db_compaction_test.cc:3479:26 (db_compaction_test+0x5cdc90) https://github.com/facebook/rocksdb/issues/6 void std::__invoke_impl(std::__invoke_other, rocksdb::DBCompactionTest_WaitForCompactShutdownWhileWaiting_Test::TestBody()::$_30&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14 (db_compaction_test+0x5cdc90) https://github.com/facebook/rocksdb/issues/7 std::__invoke_result::type std::__invoke(rocksdb::DBCompactionTest_WaitForCompactShutdownWhileWaiting_Test::TestBody()::$_30&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14 (db_compaction_test+0x5cdc90) https://github.com/facebook/rocksdb/issues/8 void std::thread::_Invoker >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:253:13 (db_compaction_test+0x5cdc90) https://github.com/facebook/rocksdb/issues/9 std::thread::_Invoker >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:260:11 (db_compaction_test+0x5cdc90) https://github.com/facebook/rocksdb/issues/10 std::thread::_State_impl > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13 (db_compaction_test+0x5cdc90) https://github.com/facebook/rocksdb/issues/11 (libstdc++.so.6+0xda6b3) Previous write of size 1 at 0x7b8c00000740 by thread T5: #0 pthread_mutex_destroy (db_compaction_test+0x46a4f8) https://github.com/facebook/rocksdb/issues/1 rocksdb::port::Mutex::~Mutex() /root/project/port/port_posix.cc:77:48 (librocksdb.so.8.4+0x7c480e) https://github.com/facebook/rocksdb/issues/2 rocksdb::InstrumentedMutex::~InstrumentedMutex() /root/project/./monitoring/instrumented_mutex.h:20:7 (librocksdb.so.8.4+0x41fda6) https://github.com/facebook/rocksdb/issues/3 rocksdb::DBImpl::~DBImpl() /root/project/db/db_impl/db_impl.cc:755:1 (librocksdb.so.8.4+0x41fda6) https://github.com/facebook/rocksdb/issues/4 rocksdb::DBImpl::~DBImpl() /root/project/db/db_impl/db_impl.cc:737:19 (librocksdb.so.8.4+0x4203d9) https://github.com/facebook/rocksdb/issues/5 rocksdb::DBTestBase::Close() /root/project/db/db_test_util.cc:670:3 (librocksdb_test_debug.so+0x57413) https://github.com/facebook/rocksdb/issues/6 rocksdb::DBCompactionTest_WaitForCompactShutdownWhileWaiting_Test::TestBody()::$_31::operator()() const /root/project/db/db_compaction_test.cc:3485:49 (db_compaction_test+0x5cdf03) https://github.com/facebook/rocksdb/issues/7 void std::__invoke_impl(std::__invoke_other, rocksdb::DBCompactionTest_WaitForCompactShutdownWhileWaiting_Test::TestBody()::$_31&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14 (db_compaction_test+0x5cdf03) https://github.com/facebook/rocksdb/issues/8 std::__invoke_result::type std::__invoke(rocksdb::DBCompactionTest_WaitForCompactShutdownWhileWaiting_Test::TestBody()::$_31&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14 (db_compaction_test+0x5cdf03) https://github.com/facebook/rocksdb/issues/9 void std::thread::_Invoker >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:253:13 (db_compaction_test+0x5cdf03) https://github.com/facebook/rocksdb/issues/10 std::thread::_Invoker >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:260:11 (db_compaction_test+0x5cdf03) https://github.com/facebook/rocksdb/issues/11 std::thread::_State_impl > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13 (db_compaction_test+0x5cdf03) https://github.com/facebook/rocksdb/issues/12 (libstdc++.so.6+0xda6b3) ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/11488 Test Plan: ``` COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 db_compaction_test gtest-parallel --repeat=10000 ./db_compaction_test --gtest_filter="*WaitForCompactShutdownWhileWaiting*" -w200 ``` Reviewed By: jaykorean Differential Revision: D46293891 Pulled By: cbi42 fbshipit-source-id: 8ca259cb1e09a9e4f4095b2d084f2ba92b710b97 --- db/db_compaction_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index 111964122..b2e372b91 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -3482,7 +3482,7 @@ TEST_F(DBCompactionTest, WaitForCompactShutdownWhileWaiting) { }); TEST_SYNC_POINT("DBCompactionTest::WaitForCompactShutdownWhileWaiting:1"); // Shutdown after wait started, but before the compaction finishes - auto closing_thread = port::Thread([this]() { Close(); }); + auto closing_thread = port::Thread([this]() { ASSERT_OK(db_->Close()); }); waiting_for_compaction_thread.join(); closing_thread.join();