Speed up BlockTest.BlockReadAmpBitmap

Summary:
BlockTest.BlockReadAmpBitmap is too slow and times out in some environments. Speed it up by:
(1) improve the way the verification is done. With this it is 5 times faster
(2) run fewer tests for large blocks. This cut it down by another 10 times.
Now it can finish in similar time as other tests.
Closes https://github.com/facebook/rocksdb/pull/3313

Differential Revision: D6643711

Pulled By: siying

fbshipit-source-id: c2397d666eab5421a78ca87e1e45491e0f832a6d
main
Siying Dong 7 years ago committed by Facebook Github Bot
parent b5c99cc908
commit ccc095a016
  1. 4
      port/win/io_win.cc
  2. 44
      table/block_test.cc

@ -961,9 +961,7 @@ Status WinWritableFile::Sync() {
return SyncImpl(); return SyncImpl();
} }
Status WinWritableFile::Fsync() { Status WinWritableFile::Fsync() { return SyncImpl(); }
return SyncImpl();
}
bool WinWritableFile::IsSyncThreadSafe() const { return true; } bool WinWritableFile::IsSyncThreadSafe() const { return true; }

@ -229,18 +229,47 @@ class BlockReadAmpBitmapSlowAndAccurate {
marked_ranges_.emplace(end_offset, start_offset); marked_ranges_.emplace(end_offset, start_offset);
} }
void ResetCheckSequence() { iter_valid_ = false; }
// Return true if any byte in this range was Marked // Return true if any byte in this range was Marked
// This does linear search from the previous position. When calling
// multiple times, `offset` needs to be incremental to get correct results.
// Call ResetCheckSequence() to reset it.
bool IsPinMarked(size_t offset) { bool IsPinMarked(size_t offset) {
auto it = marked_ranges_.lower_bound( if (iter_valid_) {
// Has existing iterator, try linear search from
// the iterator.
for (int i = 0; i < 64; i++) {
if (offset < iter_->second) {
return false;
}
if (offset <= iter_->first) {
return true;
}
iter_++;
if (iter_ == marked_ranges_.end()) {
iter_valid_ = false;
return false;
}
}
}
// Initial call or have linear searched too many times.
// Do binary search.
iter_ = marked_ranges_.lower_bound(
std::make_pair(offset, static_cast<size_t>(0))); std::make_pair(offset, static_cast<size_t>(0)));
if (it == marked_ranges_.end()) { if (iter_ == marked_ranges_.end()) {
iter_valid_ = false;
return false; return false;
} }
return offset <= it->first && offset >= it->second; iter_valid_ = true;
return offset <= iter_->first && offset >= iter_->second;
} }
private: private:
std::set<std::pair<size_t, size_t>> marked_ranges_; std::set<std::pair<size_t, size_t>> marked_ranges_;
std::set<std::pair<size_t, size_t>>::iterator iter_;
bool iter_valid_ = false;
}; };
TEST_F(BlockTest, BlockReadAmpBitmap) { TEST_F(BlockTest, BlockReadAmpBitmap) {
@ -260,9 +289,7 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
1024 * 4, // 4 KB 1024 * 4, // 4 KB
1024 * 10, // 10 KB 1024 * 10, // 10 KB
1024 * 50, // 50 KB 1024 * 50, // 50 KB
1024 * 1024, // 1 MB 1024 * 1024 * 4, // 5 MB
1024 * 1024 * 4, // 4 MB
1024 * 1024 * 50, // 10 MB
777, 777,
124653, 124653,
}; };
@ -278,10 +305,6 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
if (block_size % kBytesPerBit != 0) { if (block_size % kBytesPerBit != 0) {
needed_bits++; needed_bits++;
} }
size_t bitmap_size = needed_bits / 32;
if (needed_bits % 32 != 0) {
bitmap_size++;
}
ASSERT_EQ(stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES), block_size); ASSERT_EQ(stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES), block_size);
@ -309,6 +332,7 @@ TEST_F(BlockTest, BlockReadAmpBitmap) {
} }
for (size_t i = 0; i < random_entries.size(); i++) { for (size_t i = 0; i < random_entries.size(); i++) {
read_amp_slow_and_accurate.ResetCheckSequence();
auto &current_entry = random_entries[rnd.Next() % random_entries.size()]; auto &current_entry = random_entries[rnd.Next() % random_entries.size()];
read_amp_bitmap.Mark(static_cast<uint32_t>(current_entry.first), read_amp_bitmap.Mark(static_cast<uint32_t>(current_entry.first),

Loading…
Cancel
Save