fix issue 10751 (#10765)

Summary:
Fix https://github.com/facebook/rocksdb/issues/10751 where a stalled write could be blocked forever when DB shutdown.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10765

Reviewed By: ajkr

Differential Revision: D40110069

Pulled By: ajkr

fbshipit-source-id: 598c05777db9be85913a0a85e421b3295ecdff5e
main
Qingping Wang 2 years ago committed by Facebook GitHub Bot
parent c401f285c3
commit a45e6878f3
  1. 13
      db/db_impl/db_impl_write.cc
  2. 40
      db/db_test.cc

@ -1796,7 +1796,8 @@ Status DBImpl::DelayWrite(uint64_t num_bytes,
// might wait here indefinitely as the background compaction may never // might wait here indefinitely as the background compaction may never
// finish successfully, resulting in the stall condition lasting // finish successfully, resulting in the stall condition lasting
// indefinitely // indefinitely
while (error_handler_.GetBGError().ok() && write_controller_.IsStopped()) { while (error_handler_.GetBGError().ok() && write_controller_.IsStopped() &&
!shutting_down_.load(std::memory_order_relaxed)) {
if (write_options.no_slowdown) { if (write_options.no_slowdown) {
return Status::Incomplete("Write stall"); return Status::Incomplete("Write stall");
} }
@ -1822,9 +1823,13 @@ Status DBImpl::DelayWrite(uint64_t num_bytes,
// proceed // proceed
Status s; Status s;
if (write_controller_.IsStopped()) { if (write_controller_.IsStopped()) {
// If writes are still stopped, it means we bailed due to a background if (!shutting_down_.load(std::memory_order_relaxed)) {
// error // If writes are still stopped and db not shutdown, it means we bailed
s = Status::Incomplete(error_handler_.GetBGError().ToString()); // due to a background error
s = Status::Incomplete(error_handler_.GetBGError().ToString());
} else {
s = Status::ShutdownInProgress("stalled writes");
}
} }
if (error_handler_.IsDBStopped()) { if (error_handler_.IsDBStopped()) {
s = error_handler_.GetBGError(); s = error_handler_.GetBGError();

@ -7211,6 +7211,46 @@ TEST_F(DBTest, MemoryUsageWithMaxWriteBufferSizeToMaintain) {
} }
} }
TEST_F(DBTest, ShuttingDownNotBlockStalledWrites) {
Options options = CurrentOptions();
options.disable_auto_compactions = true;
Reopen(options);
Random rnd(403);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"BackgroundCallCompaction:1",
[&](void* /* arg */) { env_->SleepForMicroseconds(2 * 1000 * 1000); });
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
{{"DBImpl::DelayWrite:Wait",
"DBTest::ShuttingDownNotBlockStalledWrites"}});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
for (int i = 0; i < 20; i++) {
for (int j = 0; j < 100; j++) {
std::string value = rnd.RandomString(1000);
ASSERT_OK(Put("key_" + std::to_string(j), value));
}
Flush();
}
ASSERT_EQ(GetSstFileCount(dbname_), 20);
options.level0_stop_writes_trigger = 20;
options.disable_auto_compactions = false;
Reopen(options);
std::thread thd([&]() {
Status s = Put("key_" + std::to_string(101), "101");
ASSERT_EQ(s.code(), Status::kShutdownInProgress);
});
TEST_SYNC_POINT("DBTest::ShuttingDownNotBlockStalledWrites");
CancelAllBackgroundWork(db_, true);
LogFlush(options.info_log);
thd.join();
}
#endif #endif
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

Loading…
Cancel
Save