fix db_sst_test flakiness

Summary:
db_sst_test had been flaky occasionally in the following way: reached_max_space_on_compaction can in very rare cases be 0. This happens when the limit on maximum allowable space set using SetMaxAllowedSpaceUsage is hit during flush for all test db sizes (1,2,4,8 and 10MB).The fix clears the error returned when the the space limit is reached during flush. This ensures that the compaction call back will always be called. The runtime is increased slightly because the 1MB loop writes more data and hits the limit during multiple flushes until compaction is scheduled.
Closes https://github.com/facebook/rocksdb/pull/1861

Differential Revision: D4557396

Pulled By: lgalanis

fbshipit-source-id: ff778d1
main
Leonidas Galanis 8 years ago committed by Facebook Github Bot
parent 5f65dc8778
commit 72202962f9
  1. 5
      db/db_impl.cc
  2. 14
      db/db_sst_test.cc

@ -1971,8 +1971,9 @@ Status DBImpl::FlushMemTableToOutputFile(
sfm->OnAddFile(file_path); sfm->OnAddFile(file_path);
if (sfm->IsMaxAllowedSpaceReached() && bg_error_.ok()) { if (sfm->IsMaxAllowedSpaceReached() && bg_error_.ok()) {
bg_error_ = Status::IOError("Max allowed space was reached"); bg_error_ = Status::IOError("Max allowed space was reached");
TEST_SYNC_POINT( TEST_SYNC_POINT_CALLBACK(
"DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached"); "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
&bg_error_);
} }
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE

@ -516,18 +516,25 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
// than the limit. // than the limit.
std::vector<int> max_space_limits_mbs = {1, 2, 4, 8, 10}; std::vector<int> max_space_limits_mbs = {1, 2, 4, 8, 10};
decltype(max_space_limits_mbs)::value_type limit_mb_cb;
bool bg_error_set = false; bool bg_error_set = false;
uint64_t total_sst_files_size = 0; uint64_t total_sst_files_size = 0;
std::atomic<int> estimate_multiplier(1);
int reached_max_space_on_flush = 0; int reached_max_space_on_flush = 0;
int reached_max_space_on_compaction = 0; int reached_max_space_on_compaction = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached", "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
[&](void* arg) { [&](void* arg) {
Status* bg_error = static_cast<Status*>(arg);
bg_error_set = true; bg_error_set = true;
GetAllSSTFiles(&total_sst_files_size); GetAllSSTFiles(&total_sst_files_size);
reached_max_space_on_flush++; reached_max_space_on_flush++;
// low limit for size calculated using sst files
ASSERT_GE(total_sst_files_size, limit_mb_cb * 1024 * 1024);
// clear error to ensure compaction callback is called
*bg_error = Status::OK();
estimate_multiplier++; // used in the main loop assert
}); });
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
@ -541,6 +548,8 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
for (auto limit_mb : max_space_limits_mbs) { for (auto limit_mb : max_space_limits_mbs) {
bg_error_set = false; bg_error_set = false;
total_sst_files_size = 0; total_sst_files_size = 0;
estimate_multiplier = 1;
limit_mb_cb = limit_mb;
rocksdb::SyncPoint::GetInstance()->ClearTrace(); rocksdb::SyncPoint::GetInstance()->ClearTrace();
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_)); std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
@ -565,7 +574,8 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
// Check the estimated db size vs the db limit just to make sure we // Check the estimated db size vs the db limit just to make sure we
// dont run into an infinite loop // dont run into an infinite loop
estimated_db_size = keys_written * 60; // ~60 bytes per key estimated_db_size = keys_written * 60; // ~60 bytes per key
ASSERT_LT(estimated_db_size, limit_mb * 1024 * 1024 * 2); ASSERT_LT(estimated_db_size,
estimate_multiplier * limit_mb * 1024 * 1024 * 2);
} }
ASSERT_TRUE(bg_error_set); ASSERT_TRUE(bg_error_set);
ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024); ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024);

Loading…
Cancel
Save