NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pass rate_limiter_priority through filter block reader functions to FS (#10251)

Browse Source 
Summary:
With https://github.com/facebook/rocksdb/pull/9996 , we can pass the rate_limiter_priority to FS for most cases. This PR is to update the code path for filter block reader.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10251

Test Plan: Current unit tests should pass.

Reviewed By: pdillinger

Differential Revision: D37427667

Pulled By: gitbw95

fbshipit-source-id: 1ce5b759b136efe4cfa48a6b97e2f837ff087433
main
Bo Wang 2 years ago committed by Facebook GitHub Bot
parent 410ca2efd2
commit 8e63d90ff8
13 changed files with 182 additions and 98 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 25
      table/block_based/block_based_table_reader.cc
  2. 6
      table/block_based/block_based_table_reader.h
  3. 2
      table/block_based/block_based_table_reader_sync_and_async.h
  4. 20
      table/block_based/filter_block.h
  5. 10
      table/block_based/filter_block_reader_common.cc
  6. 6
      table/block_based/filter_block_reader_common.h
  7. 45
      table/block_based/full_filter_block.cc
  8. 24
      table/block_based/full_filter_block.h
  9. 48
      table/block_based/full_filter_block_test.cc
  10. 46
      table/block_based/partitioned_filter_block.cc
  11. 21
      table/block_based/partitioned_filter_block.h
  12. 23
      table/block_based/partitioned_filter_block_test.cc
  13. 4
      util/filter_bench.cc

25
table/block_based/block_based_table_reader.cc
Unescape View File

@ -1902,7 +1902,8 @@ bool BlockBasedTable::PrefixRangeMayMatch(
may_match = filter->RangeMayExist( may_match = filter->RangeMayExist(
read_options.iterate_upper_bound, user_key_without_ts, prefix_extractor, read_options.iterate_upper_bound, user_key_without_ts, prefix_extractor,
rep_->internal_comparator.user_comparator(), const_ikey_ptr, rep_->internal_comparator.user_comparator(), const_ikey_ptr,
&filter_checked, need_upper_bound_check, no_io, lookup_context); &filter_checked, need_upper_bound_check, no_io, lookup_context,
read_options.rate_limiter_priority);
} }
if (filter_checked) { if (filter_checked) {
@ -1974,7 +1975,8 @@ FragmentedRangeTombstoneIterator* BlockBasedTable::NewRangeTombstoneIterator(
bool BlockBasedTable::FullFilterKeyMayMatch( bool BlockBasedTable::FullFilterKeyMayMatch(
FilterBlockReader* filter, const Slice& internal_key, const bool no_io, FilterBlockReader* filter, const Slice& internal_key, const bool no_io,
const SliceTransform* prefix_extractor, GetContext* get_context, const SliceTransform* prefix_extractor, GetContext* get_context,
BlockCacheLookupContext* lookup_context) const { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const {
if (filter == nullptr) { if (filter == nullptr) {
return true; return true;
} }
@ -1984,13 +1986,15 @@ bool BlockBasedTable::FullFilterKeyMayMatch(
size_t ts_sz = rep_->internal_comparator.user_comparator()->timestamp_size(); size_t ts_sz = rep_->internal_comparator.user_comparator()->timestamp_size();
Slice user_key_without_ts = StripTimestampFromUserKey(user_key, ts_sz); Slice user_key_without_ts = StripTimestampFromUserKey(user_key, ts_sz);
if (rep_->whole_key_filtering) { if (rep_->whole_key_filtering) {
may_match = filter->KeyMayMatch(user_key_without_ts, no_io, const_ikey_ptr, may_match =
get_context, lookup_context); filter->KeyMayMatch(user_key_without_ts, no_io, const_ikey_ptr,
get_context, lookup_context, rate_limiter_priority);
} else if (!PrefixExtractorChanged(prefix_extractor) && } else if (!PrefixExtractorChanged(prefix_extractor) &&
prefix_extractor->InDomain(user_key_without_ts) && prefix_extractor->InDomain(user_key_without_ts) &&
!filter->PrefixMayMatch( !filter->PrefixMayMatch(
prefix_extractor->Transform(user_key_without_ts), no_io, prefix_extractor->Transform(user_key_without_ts), no_io,
const_ikey_ptr, get_context, lookup_context)) { const_ikey_ptr, get_context, lookup_context,
rate_limiter_priority)) {
// FIXME ^^^: there should be no reason for Get() to depend on current // FIXME ^^^: there should be no reason for Get() to depend on current
// prefix_extractor at all. It should always use table_prefix_extractor. // prefix_extractor at all. It should always use table_prefix_extractor.
may_match = false; may_match = false;
@ -2005,14 +2009,15 @@ bool BlockBasedTable::FullFilterKeyMayMatch(
void BlockBasedTable::FullFilterKeysMayMatch( void BlockBasedTable::FullFilterKeysMayMatch(
FilterBlockReader* filter, MultiGetRange* range, const bool no_io, FilterBlockReader* filter, MultiGetRange* range, const bool no_io,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
BlockCacheLookupContext* lookup_context) const { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const {
if (filter == nullptr) { if (filter == nullptr) {
return; return;
} }
uint64_t before_keys = range->KeysLeft(); uint64_t before_keys = range->KeysLeft();
assert(before_keys > 0); // Caller should ensure assert(before_keys > 0); // Caller should ensure
if (rep_->whole_key_filtering) { if (rep_->whole_key_filtering) {
filter->KeysMayMatch(range, no_io, lookup_context); filter->KeysMayMatch(range, no_io, lookup_context, rate_limiter_priority);
uint64_t after_keys = range->KeysLeft(); uint64_t after_keys = range->KeysLeft();
if (after_keys) { if (after_keys) {
RecordTick(rep_->ioptions.stats, BLOOM_FILTER_FULL_POSITIVE, after_keys); RecordTick(rep_->ioptions.stats, BLOOM_FILTER_FULL_POSITIVE, after_keys);
@ -2028,7 +2033,8 @@ void BlockBasedTable::FullFilterKeysMayMatch(
} else if (!PrefixExtractorChanged(prefix_extractor)) { } else if (!PrefixExtractorChanged(prefix_extractor)) {
// FIXME ^^^: there should be no reason for MultiGet() to depend on current // FIXME ^^^: there should be no reason for MultiGet() to depend on current
// prefix_extractor at all. It should always use table_prefix_extractor. // prefix_extractor at all. It should always use table_prefix_extractor.
filter->PrefixesMayMatch(range, prefix_extractor, false, lookup_context); filter->PrefixesMayMatch(range, prefix_extractor, false, lookup_context,
rate_limiter_priority);
RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_CHECKED, before_keys); RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_CHECKED, before_keys);
uint64_t after_keys = range->KeysLeft(); uint64_t after_keys = range->KeysLeft();
uint64_t filtered_keys = before_keys - after_keys; uint64_t filtered_keys = before_keys - after_keys;
@ -2065,7 +2071,8 @@ Status BlockBasedTable::Get(const ReadOptions& read_options, const Slice& key,
} }
TEST_SYNC_POINT("BlockBasedTable::Get:BeforeFilterMatch"); TEST_SYNC_POINT("BlockBasedTable::Get:BeforeFilterMatch");
const bool may_match = FullFilterKeyMayMatch( const bool may_match = FullFilterKeyMayMatch(
filter, key, no_io, prefix_extractor, get_context, &lookup_context); filter, key, no_io, prefix_extractor, get_context, &lookup_context,
read_options.rate_limiter_priority);
TEST_SYNC_POINT("BlockBasedTable::Get:AfterFilterMatch"); TEST_SYNC_POINT("BlockBasedTable::Get:AfterFilterMatch");
if (!may_match) { if (!may_match) {
RecordTick(rep_->ioptions.stats, BLOOM_FILTER_USEFUL); RecordTick(rep_->ioptions.stats, BLOOM_FILTER_USEFUL);

6
table/block_based/block_based_table_reader.h
Unescape View File

@ -454,12 +454,14 @@ class BlockBasedTable : public TableReader {
const bool no_io, const bool no_io,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
GetContext* get_context, GetContext* get_context,
BlockCacheLookupContext* lookup_context) const; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const;
void FullFilterKeysMayMatch(FilterBlockReader* filter, MultiGetRange* range, void FullFilterKeysMayMatch(FilterBlockReader* filter, MultiGetRange* range,
const bool no_io, const bool no_io,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
BlockCacheLookupContext* lookup_context) const; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const;
// If force_direct_prefetch is true, always prefetching to RocksDB // If force_direct_prefetch is true, always prefetching to RocksDB
// buffer, rather than calling RandomAccessFile::Prefetch(). // buffer, rather than calling RandomAccessFile::Prefetch().

2
table/block_based/block_based_table_reader_sync_and_async.h
Unescape View File

@ -335,7 +335,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet)
TableReaderCaller::kUserMultiGet, tracing_mget_id, TableReaderCaller::kUserMultiGet, tracing_mget_id,
/*_get_from_user_specified_snapshot=*/read_options.snapshot != nullptr}; /*_get_from_user_specified_snapshot=*/read_options.snapshot != nullptr};
FullFilterKeysMayMatch(filter, &sst_file_range, no_io, prefix_extractor, FullFilterKeysMayMatch(filter, &sst_file_range, no_io, prefix_extractor,
&lookup_context); &lookup_context, read_options.rate_limiter_priority);
if (!sst_file_range.empty()) { if (!sst_file_range.empty()) {
IndexBlockIter iiter_on_stack; IndexBlockIter iiter_on_stack;

20
table/block_based/filter_block.h
Unescape View File

@ -112,16 +112,18 @@ class FilterBlockReader {
virtual bool KeyMayMatch(const Slice& key, const bool no_io, virtual bool KeyMayMatch(const Slice& key, const bool no_io,
const Slice* const const_ikey_ptr, const Slice* const const_ikey_ptr,
GetContext* get_context, GetContext* get_context,
BlockCacheLookupContext* lookup_context) = 0; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) = 0;
virtual void KeysMayMatch(MultiGetRange* range, const bool no_io, virtual void KeysMayMatch(MultiGetRange* range, const bool no_io,
BlockCacheLookupContext* lookup_context) { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
for (auto iter = range->begin(); iter != range->end(); ++iter) { for (auto iter = range->begin(); iter != range->end(); ++iter) {
const Slice ukey_without_ts = iter->ukey_without_ts; const Slice ukey_without_ts = iter->ukey_without_ts;
const Slice ikey = iter->ikey; const Slice ikey = iter->ikey;
GetContext* const get_context = iter->get_context; GetContext* const get_context = iter->get_context;
if (!KeyMayMatch(ukey_without_ts, no_io, &ikey, get_context, if (!KeyMayMatch(ukey_without_ts, no_io, &ikey, get_context,
lookup_context)) { lookup_context, rate_limiter_priority)) {
range->SkipKey(iter); range->SkipKey(iter);
} }
} }
@ -133,19 +135,22 @@ class FilterBlockReader {
virtual bool PrefixMayMatch(const Slice& prefix, const bool no_io, virtual bool PrefixMayMatch(const Slice& prefix, const bool no_io,
const Slice* const const_ikey_ptr, const Slice* const const_ikey_ptr,
GetContext* get_context, GetContext* get_context,
BlockCacheLookupContext* lookup_context) = 0; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) = 0;
virtual void PrefixesMayMatch(MultiGetRange* range, virtual void PrefixesMayMatch(MultiGetRange* range,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
const bool no_io, const bool no_io,
BlockCacheLookupContext* lookup_context) { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
for (auto iter = range->begin(); iter != range->end(); ++iter) { for (auto iter = range->begin(); iter != range->end(); ++iter) {
const Slice ukey_without_ts = iter->ukey_without_ts; const Slice ukey_without_ts = iter->ukey_without_ts;
const Slice ikey = iter->ikey; const Slice ikey = iter->ikey;
GetContext* const get_context = iter->get_context; GetContext* const get_context = iter->get_context;
if (prefix_extractor->InDomain(ukey_without_ts) && if (prefix_extractor->InDomain(ukey_without_ts) &&
!PrefixMayMatch(prefix_extractor->Transform(ukey_without_ts), no_io, !PrefixMayMatch(prefix_extractor->Transform(ukey_without_ts), no_io,
&ikey, get_context, lookup_context)) { &ikey, get_context, lookup_context,
rate_limiter_priority)) {
range->SkipKey(iter); range->SkipKey(iter);
} }
} }
@ -170,7 +175,8 @@ class FilterBlockReader {
const Slice* const const_ikey_ptr, const Slice* const const_ikey_ptr,
bool* filter_checked, bool need_upper_bound_check, bool* filter_checked, bool need_upper_bound_check,
bool no_io, bool no_io,
BlockCacheLookupContext* lookup_context) = 0; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) = 0;
}; };
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

10
table/block_based/filter_block_reader_common.cc
Unescape View File

@ -67,7 +67,8 @@ template <typename TBlocklike>
Status FilterBlockReaderCommon<TBlocklike>::GetOrReadFilterBlock( Status FilterBlockReaderCommon<TBlocklike>::GetOrReadFilterBlock(
bool no_io, GetContext* get_context, bool no_io, GetContext* get_context,
BlockCacheLookupContext* lookup_context, BlockCacheLookupContext* lookup_context,
CachableEntry<TBlocklike>* filter_block, BlockType block_type) const { CachableEntry<TBlocklike>* filter_block, BlockType block_type,
Env::IOPriority rate_limiter_priority) const {
assert(filter_block); assert(filter_block);
if (!filter_block_.IsEmpty()) { if (!filter_block_.IsEmpty()) {
@ -76,6 +77,7 @@ Status FilterBlockReaderCommon<TBlocklike>::GetOrReadFilterBlock(
} }
ReadOptions read_options; ReadOptions read_options;
read_options.rate_limiter_priority = rate_limiter_priority;
if (no_io) { if (no_io) {
read_options.read_tier = kBlockCacheTier; read_options.read_tier = kBlockCacheTier;
} }
@ -100,7 +102,8 @@ bool FilterBlockReaderCommon<TBlocklike>::RangeMayExist(
const SliceTransform* prefix_extractor, const Comparator* comparator, const SliceTransform* prefix_extractor, const Comparator* comparator,
const Slice* const const_ikey_ptr, bool* filter_checked, const Slice* const const_ikey_ptr, bool* filter_checked,
bool need_upper_bound_check, bool no_io, bool need_upper_bound_check, bool no_io,
BlockCacheLookupContext* lookup_context) { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
if (!prefix_extractor || !prefix_extractor->InDomain(user_key_without_ts)) { if (!prefix_extractor || !prefix_extractor->InDomain(user_key_without_ts)) {
*filter_checked = false; *filter_checked = false;
return true; return true;
@ -113,7 +116,8 @@ bool FilterBlockReaderCommon<TBlocklike>::RangeMayExist(
} else { } else {
*filter_checked = true; *filter_checked = true;
return PrefixMayMatch(prefix, no_io, const_ikey_ptr, return PrefixMayMatch(prefix, no_io, const_ikey_ptr,
/* get_context */ nullptr, lookup_context); /* get_context */ nullptr, lookup_context,
rate_limiter_priority);
} }
} }

6
table/block_based/filter_block_reader_common.h
Unescape View File

@ -40,7 +40,8 @@ class FilterBlockReaderCommon : public FilterBlockReader {
const Comparator* comparator, const Comparator* comparator,
const Slice* const const_ikey_ptr, bool* filter_checked, const Slice* const const_ikey_ptr, bool* filter_checked,
bool need_upper_bound_check, bool no_io, bool need_upper_bound_check, bool no_io,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
protected: protected:
static Status ReadFilterBlock(const BlockBasedTable* table, static Status ReadFilterBlock(const BlockBasedTable* table,
@ -59,7 +60,8 @@ class FilterBlockReaderCommon : public FilterBlockReader {
Status GetOrReadFilterBlock(bool no_io, GetContext* get_context, Status GetOrReadFilterBlock(bool no_io, GetContext* get_context,
BlockCacheLookupContext* lookup_context, BlockCacheLookupContext* lookup_context,
CachableEntry<TBlocklike>* filter_block, CachableEntry<TBlocklike>* filter_block,
BlockType block_type) const; BlockType block_type,
Env::IOPriority rate_limiter_priority) const;
size_t ApproximateFilterBlockMemoryUsage() const; size_t ApproximateFilterBlockMemoryUsage() const;

45
table/block_based/full_filter_block.cc
Unescape View File

@ -124,13 +124,16 @@ FullFilterBlockReader::FullFilterBlockReader(
: FilterBlockReaderCommon(t, std::move(filter_block)) { : FilterBlockReaderCommon(t, std::move(filter_block)) {
} }
bool FullFilterBlockReader::KeyMayMatch( bool FullFilterBlockReader::KeyMayMatch(const Slice& key, const bool no_io,
const Slice& key, const bool no_io, const Slice* const /*const_ikey_ptr*/, const Slice* const /*const_ikey_ptr*/,
GetContext* get_context, BlockCacheLookupContext* lookup_context) { GetContext* get_context,
BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
if (!whole_key_filtering()) { if (!whole_key_filtering()) {
return true; return true;
} }
return MayMatch(key, no_io, get_context, lookup_context); return MayMatch(key, no_io, get_context, lookup_context,
rate_limiter_priority);
} }
std::unique_ptr<FilterBlockReader> FullFilterBlockReader::Create( std::unique_ptr<FilterBlockReader> FullFilterBlockReader::Create(
@ -163,17 +166,21 @@ std::unique_ptr<FilterBlockReader> FullFilterBlockReader::Create(
bool FullFilterBlockReader::PrefixMayMatch( bool FullFilterBlockReader::PrefixMayMatch(
const Slice& prefix, const bool no_io, const Slice& prefix, const bool no_io,
const Slice* const /*const_ikey_ptr*/, GetContext* get_context, const Slice* const /*const_ikey_ptr*/, GetContext* get_context,
BlockCacheLookupContext* lookup_context) { BlockCacheLookupContext* lookup_context,
return MayMatch(prefix, no_io, get_context, lookup_context); Env::IOPriority rate_limiter_priority) {
return MayMatch(prefix, no_io, get_context, lookup_context,
rate_limiter_priority);
} }
bool FullFilterBlockReader::MayMatch( bool FullFilterBlockReader::MayMatch(
const Slice& entry, bool no_io, GetContext* get_context, const Slice& entry, bool no_io, GetContext* get_context,
BlockCacheLookupContext* lookup_context) const { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const {
CachableEntry<ParsedFullFilterBlock> filter_block; CachableEntry<ParsedFullFilterBlock> filter_block;
const Status s = GetOrReadFilterBlock(no_io, get_context, lookup_context, const Status s =
&filter_block, BlockType::kFilter); GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block,
BlockType::kFilter, rate_limiter_priority);
if (!s.ok()) { if (!s.ok()) {
IGNORE_STATUS_IF_ERROR(s); IGNORE_STATUS_IF_ERROR(s);
return true; return true;
@ -198,29 +205,33 @@ bool FullFilterBlockReader::MayMatch(
void FullFilterBlockReader::KeysMayMatch( void FullFilterBlockReader::KeysMayMatch(
MultiGetRange* range, const bool no_io, MultiGetRange* range, const bool no_io,
BlockCacheLookupContext* lookup_context) { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
if (!whole_key_filtering()) { if (!whole_key_filtering()) {
// Simply return. Don't skip any key - consider all keys as likely to be // Simply return. Don't skip any key - consider all keys as likely to be
// present // present
return; return;
} }
MayMatch(range, no_io, nullptr, lookup_context); MayMatch(range, no_io, nullptr, lookup_context, rate_limiter_priority);
} }
void FullFilterBlockReader::PrefixesMayMatch( void FullFilterBlockReader::PrefixesMayMatch(
MultiGetRange* range, const SliceTransform* prefix_extractor, MultiGetRange* range, const SliceTransform* prefix_extractor,
const bool no_io, BlockCacheLookupContext* lookup_context) { const bool no_io, BlockCacheLookupContext* lookup_context,
MayMatch(range, no_io, prefix_extractor, lookup_context); Env::IOPriority rate_limiter_priority) {
MayMatch(range, no_io, prefix_extractor, lookup_context,
rate_limiter_priority);
} }
void FullFilterBlockReader::MayMatch( void FullFilterBlockReader::MayMatch(
MultiGetRange* range, bool no_io, const SliceTransform* prefix_extractor, MultiGetRange* range, bool no_io, const SliceTransform* prefix_extractor,
BlockCacheLookupContext* lookup_context) const { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const {
CachableEntry<ParsedFullFilterBlock> filter_block; CachableEntry<ParsedFullFilterBlock> filter_block;
const Status s = const Status s = GetOrReadFilterBlock(
GetOrReadFilterBlock(no_io, range->begin()->get_context, lookup_context, no_io, range->begin()->get_context, lookup_context, &filter_block,
&filter_block, BlockType::kFilter); BlockType::kFilter, rate_limiter_priority);
if (!s.ok()) { if (!s.ok()) {
IGNORE_STATUS_IF_ERROR(s); IGNORE_STATUS_IF_ERROR(s);
return; return;

24
table/block_based/full_filter_block.h
Unescape View File

@ -107,34 +107,40 @@ class FullFilterBlockReader
bool KeyMayMatch(const Slice& key, const bool no_io, bool KeyMayMatch(const Slice& key, const bool no_io,
const Slice* const const_ikey_ptr, GetContext* get_context, const Slice* const const_ikey_ptr, GetContext* get_context,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
bool PrefixMayMatch(const Slice& prefix, const bool no_io, bool PrefixMayMatch(const Slice& prefix, const bool no_io,
const Slice* const const_ikey_ptr, const Slice* const const_ikey_ptr,
GetContext* get_context, GetContext* get_context,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
void KeysMayMatch(MultiGetRange* range, const bool no_io, void KeysMayMatch(MultiGetRange* range, const bool no_io,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
// Used in partitioned filter code // Used in partitioned filter code
void KeysMayMatch2(MultiGetRange* range, void KeysMayMatch2(MultiGetRange* range,
const SliceTransform* /*prefix_extractor*/, const SliceTransform* /*prefix_extractor*/,
const bool no_io, const bool no_io, BlockCacheLookupContext* lookup_context,
BlockCacheLookupContext* lookup_context) { Env::IOPriority rate_limiter_priority) {
KeysMayMatch(range, no_io, lookup_context); KeysMayMatch(range, no_io, lookup_context, rate_limiter_priority);
} }
void PrefixesMayMatch(MultiGetRange* range, void PrefixesMayMatch(MultiGetRange* range,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
const bool no_io, const bool no_io,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
size_t ApproximateMemoryUsage() const override; size_t ApproximateMemoryUsage() const override;
private: private:
bool MayMatch(const Slice& entry, bool no_io, GetContext* get_context, bool MayMatch(const Slice& entry, bool no_io, GetContext* get_context,
BlockCacheLookupContext* lookup_context) const; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const;
void MayMatch(MultiGetRange* range, bool no_io, void MayMatch(MultiGetRange* range, bool no_io,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
BlockCacheLookupContext* lookup_context) const; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) const;
}; };
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

48
table/block_based/full_filter_block_test.cc
Unescape View File

@ -118,7 +118,7 @@ TEST_F(PluginFullFilterBlockTest, PluginEmptyBuilder) {
ASSERT_TRUE(reader.KeyMayMatch("foo", ASSERT_TRUE(reader.KeyMayMatch("foo",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr, Env::IO_TOTAL));
} }
TEST_F(PluginFullFilterBlockTest, PluginSingleChunk) { TEST_F(PluginFullFilterBlockTest, PluginSingleChunk) {
@ -136,34 +136,42 @@ TEST_F(PluginFullFilterBlockTest, PluginSingleChunk) {
nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */);
FullFilterBlockReader reader(table_.get(), std::move(block)); FullFilterBlockReader reader(table_.get(), std::move(block));
Env::IOPriority rate_limiter_priority = Env::IO_TOTAL;
ASSERT_TRUE(reader.KeyMayMatch("foo", ASSERT_TRUE(reader.KeyMayMatch("foo",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("bar", ASSERT_TRUE(reader.KeyMayMatch("bar",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("box", ASSERT_TRUE(reader.KeyMayMatch("box",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("hello", ASSERT_TRUE(reader.KeyMayMatch("hello",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("foo", ASSERT_TRUE(reader.KeyMayMatch("foo",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(!reader.KeyMayMatch("missing", ASSERT_TRUE(!reader.KeyMayMatch("missing",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(!reader.KeyMayMatch("other", ASSERT_TRUE(!reader.KeyMayMatch("other",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
} }
class FullFilterBlockTest : public mock::MockBlockBasedTableTester, class FullFilterBlockTest : public mock::MockBlockBasedTableTester,
@ -188,7 +196,7 @@ TEST_F(FullFilterBlockTest, EmptyBuilder) {
ASSERT_TRUE(reader.KeyMayMatch("foo", ASSERT_TRUE(reader.KeyMayMatch("foo",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr, Env::IO_TOTAL));
} }
class CountUniqueFilterBitsBuilderWrapper : public FilterBitsBuilder { class CountUniqueFilterBitsBuilderWrapper : public FilterBitsBuilder {
@ -285,34 +293,42 @@ TEST_F(FullFilterBlockTest, SingleChunk) {
nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */);
FullFilterBlockReader reader(table_.get(), std::move(block)); FullFilterBlockReader reader(table_.get(), std::move(block));
Env::IOPriority rate_limiter_priority = Env::IO_TOTAL;
ASSERT_TRUE(reader.KeyMayMatch("foo", ASSERT_TRUE(reader.KeyMayMatch("foo",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("bar", ASSERT_TRUE(reader.KeyMayMatch("bar",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("box", ASSERT_TRUE(reader.KeyMayMatch("box",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("hello", ASSERT_TRUE(reader.KeyMayMatch("hello",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(reader.KeyMayMatch("foo", ASSERT_TRUE(reader.KeyMayMatch("foo",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(!reader.KeyMayMatch("missing", ASSERT_TRUE(!reader.KeyMayMatch("missing",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
ASSERT_TRUE(!reader.KeyMayMatch("other", ASSERT_TRUE(!reader.KeyMayMatch("other",
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

46
table/block_based/partitioned_filter_block.cc
Unescape View File

@ -217,41 +217,46 @@ std::unique_ptr<FilterBlockReader> PartitionedFilterBlockReader::Create(
bool PartitionedFilterBlockReader::KeyMayMatch( bool PartitionedFilterBlockReader::KeyMayMatch(
const Slice& key, const bool no_io, const Slice* const const_ikey_ptr, const Slice& key, const bool no_io, const Slice* const const_ikey_ptr,
GetContext* get_context, BlockCacheLookupContext* lookup_context) { GetContext* get_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
assert(const_ikey_ptr != nullptr); assert(const_ikey_ptr != nullptr);
if (!whole_key_filtering()) { if (!whole_key_filtering()) {
return true; return true;
} }
return MayMatch(key, no_io, const_ikey_ptr, get_context, lookup_context, return MayMatch(key, no_io, const_ikey_ptr, get_context, lookup_context,
&FullFilterBlockReader::KeyMayMatch); rate_limiter_priority, &FullFilterBlockReader::KeyMayMatch);
} }
void PartitionedFilterBlockReader::KeysMayMatch( void PartitionedFilterBlockReader::KeysMayMatch(
MultiGetRange* range, const bool no_io, MultiGetRange* range, const bool no_io,
BlockCacheLookupContext* lookup_context) { BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
if (!whole_key_filtering()) { if (!whole_key_filtering()) {
return; // Any/all may match return; // Any/all may match
} }
MayMatch(range, nullptr, no_io, lookup_context, MayMatch(range, nullptr, no_io, lookup_context, rate_limiter_priority,
&FullFilterBlockReader::KeysMayMatch2); &FullFilterBlockReader::KeysMayMatch2);
} }
bool PartitionedFilterBlockReader::PrefixMayMatch( bool PartitionedFilterBlockReader::PrefixMayMatch(
const Slice& prefix, const bool no_io, const Slice* const const_ikey_ptr, const Slice& prefix, const bool no_io, const Slice* const const_ikey_ptr,
GetContext* get_context, BlockCacheLookupContext* lookup_context) { GetContext* get_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
assert(const_ikey_ptr != nullptr); assert(const_ikey_ptr != nullptr);
return MayMatch(prefix, no_io, const_ikey_ptr, get_context, lookup_context, return MayMatch(prefix, no_io, const_ikey_ptr, get_context, lookup_context,
rate_limiter_priority,
&FullFilterBlockReader::PrefixMayMatch); &FullFilterBlockReader::PrefixMayMatch);
} }
void PartitionedFilterBlockReader::PrefixesMayMatch( void PartitionedFilterBlockReader::PrefixesMayMatch(
MultiGetRange* range, const SliceTransform* prefix_extractor, MultiGetRange* range, const SliceTransform* prefix_extractor,
const bool no_io, BlockCacheLookupContext* lookup_context) { const bool no_io, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) {
assert(prefix_extractor); assert(prefix_extractor);
MayMatch(range, prefix_extractor, no_io, lookup_context, MayMatch(range, prefix_extractor, no_io, lookup_context,
&FullFilterBlockReader::PrefixesMayMatch); rate_limiter_priority, &FullFilterBlockReader::PrefixesMayMatch);
} }
BlockHandle PartitionedFilterBlockReader::GetFilterPartitionHandle( BlockHandle PartitionedFilterBlockReader::GetFilterPartitionHandle(
@ -316,11 +321,12 @@ Status PartitionedFilterBlockReader::GetFilterPartitionBlock(
bool PartitionedFilterBlockReader::MayMatch( bool PartitionedFilterBlockReader::MayMatch(
const Slice& slice, bool no_io, const Slice* const_ikey_ptr, const Slice& slice, bool no_io, const Slice* const_ikey_ptr,
GetContext* get_context, BlockCacheLookupContext* lookup_context, GetContext* get_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority,
FilterFunction filter_function) const { FilterFunction filter_function) const {
CachableEntry<Block> filter_block; CachableEntry<Block> filter_block;
Status s = Status s = GetOrReadFilterBlock(
GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block, no_io, get_context, lookup_context, &filter_block,
BlockType::kFilterPartitionIndex); BlockType::kFilterPartitionIndex, rate_limiter_priority);
if (UNLIKELY(!s.ok())) { if (UNLIKELY(!s.ok())) {
IGNORE_STATUS_IF_ERROR(s); IGNORE_STATUS_IF_ERROR(s);
return true; return true;
@ -347,17 +353,19 @@ bool PartitionedFilterBlockReader::MayMatch(
FullFilterBlockReader filter_partition(table(), FullFilterBlockReader filter_partition(table(),
std::move(filter_partition_block)); std::move(filter_partition_block));
return (filter_partition.*filter_function)(slice, no_io, const_ikey_ptr, return (filter_partition.*filter_function)(slice, no_io, const_ikey_ptr,
get_context, lookup_context); get_context, lookup_context,
rate_limiter_priority);
} }
void PartitionedFilterBlockReader::MayMatch( void PartitionedFilterBlockReader::MayMatch(
MultiGetRange* range, const SliceTransform* prefix_extractor, bool no_io, MultiGetRange* range, const SliceTransform* prefix_extractor, bool no_io,
BlockCacheLookupContext* lookup_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority,
FilterManyFunction filter_function) const { FilterManyFunction filter_function) const {
CachableEntry<Block> filter_block; CachableEntry<Block> filter_block;
Status s = Status s = GetOrReadFilterBlock(
GetOrReadFilterBlock(no_io, range->begin()->get_context, lookup_context, no_io, range->begin()->get_context, lookup_context, &filter_block,
&filter_block, BlockType::kFilterPartitionIndex); BlockType::kFilterPartitionIndex, rate_limiter_priority);
if (UNLIKELY(!s.ok())) { if (UNLIKELY(!s.ok())) {
IGNORE_STATUS_IF_ERROR(s); IGNORE_STATUS_IF_ERROR(s);
return; // Any/all may match return; // Any/all may match
@ -381,7 +389,7 @@ void PartitionedFilterBlockReader::MayMatch(
this_filter_handle != prev_filter_handle) { this_filter_handle != prev_filter_handle) {
MultiGetRange subrange(*range, start_iter_same_handle, iter); MultiGetRange subrange(*range, start_iter_same_handle, iter);
MayMatchPartition(&subrange, prefix_extractor, prev_filter_handle, no_io, MayMatchPartition(&subrange, prefix_extractor, prev_filter_handle, no_io,
lookup_context, filter_function); lookup_context, rate_limiter_priority, filter_function);
range->AddSkipsFrom(subrange); range->AddSkipsFrom(subrange);
start_iter_same_handle = iter; start_iter_same_handle = iter;
} }
@ -397,7 +405,7 @@ void PartitionedFilterBlockReader::MayMatch(
if (!prev_filter_handle.IsNull()) { if (!prev_filter_handle.IsNull()) {
MultiGetRange subrange(*range, start_iter_same_handle, range->end()); MultiGetRange subrange(*range, start_iter_same_handle, range->end());
MayMatchPartition(&subrange, prefix_extractor, prev_filter_handle, no_io, MayMatchPartition(&subrange, prefix_extractor, prev_filter_handle, no_io,
lookup_context, filter_function); lookup_context, rate_limiter_priority, filter_function);
range->AddSkipsFrom(subrange); range->AddSkipsFrom(subrange);
} }
} }
@ -406,6 +414,7 @@ void PartitionedFilterBlockReader::MayMatchPartition(
MultiGetRange* range, const SliceTransform* prefix_extractor, MultiGetRange* range, const SliceTransform* prefix_extractor,
BlockHandle filter_handle, bool no_io, BlockHandle filter_handle, bool no_io,
BlockCacheLookupContext* lookup_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority,
FilterManyFunction filter_function) const { FilterManyFunction filter_function) const {
CachableEntry<ParsedFullFilterBlock> filter_partition_block; CachableEntry<ParsedFullFilterBlock> filter_partition_block;
Status s = GetFilterPartitionBlock( Status s = GetFilterPartitionBlock(
@ -419,7 +428,7 @@ void PartitionedFilterBlockReader::MayMatchPartition(
FullFilterBlockReader filter_partition(table(), FullFilterBlockReader filter_partition(table(),
std::move(filter_partition_block)); std::move(filter_partition_block));
(filter_partition.*filter_function)(range, prefix_extractor, no_io, (filter_partition.*filter_function)(range, prefix_extractor, no_io,
lookup_context); lookup_context, rate_limiter_priority);
} }
size_t PartitionedFilterBlockReader::ApproximateMemoryUsage() const { size_t PartitionedFilterBlockReader::ApproximateMemoryUsage() const {
@ -447,7 +456,8 @@ Status PartitionedFilterBlockReader::CacheDependencies(const ReadOptions& ro,
Status s = GetOrReadFilterBlock(false /* no_io */, nullptr /* get_context */, Status s = GetOrReadFilterBlock(false /* no_io */, nullptr /* get_context */,
&lookup_context, &filter_block, &lookup_context, &filter_block,
BlockType::kFilterPartitionIndex); BlockType::kFilterPartitionIndex,
ro.rate_limiter_priority);
if (!s.ok()) { if (!s.ok()) {
ROCKS_LOG_ERROR(rep->ioptions.logger, ROCKS_LOG_ERROR(rep->ioptions.logger,
"Error retrieving top-level filter block while trying to " "Error retrieving top-level filter block while trying to "

21
table/block_based/partitioned_filter_block.h
Unescape View File

@ -111,18 +111,22 @@ class PartitionedFilterBlockReader : public FilterBlockReaderCommon<Block> {
bool KeyMayMatch(const Slice& key, const bool no_io, bool KeyMayMatch(const Slice& key, const bool no_io,
const Slice* const const_ikey_ptr, GetContext* get_context, const Slice* const const_ikey_ptr, GetContext* get_context,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
void KeysMayMatch(MultiGetRange* range, const bool no_io, void KeysMayMatch(MultiGetRange* range, const bool no_io,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
bool PrefixMayMatch(const Slice& prefix, const bool no_io, bool PrefixMayMatch(const Slice& prefix, const bool no_io,
const Slice* const const_ikey_ptr, const Slice* const const_ikey_ptr,
GetContext* get_context, GetContext* get_context,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
void PrefixesMayMatch(MultiGetRange* range, void PrefixesMayMatch(MultiGetRange* range,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
const bool no_io, const bool no_io,
BlockCacheLookupContext* lookup_context) override; BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority) override;
size_t ApproximateMemoryUsage() const override; size_t ApproximateMemoryUsage() const override;
@ -137,21 +141,26 @@ class PartitionedFilterBlockReader : public FilterBlockReaderCommon<Block> {
using FilterFunction = bool (FullFilterBlockReader::*)( using FilterFunction = bool (FullFilterBlockReader::*)(
const Slice& slice, const bool no_io, const Slice* const const_ikey_ptr, const Slice& slice, const bool no_io, const Slice* const const_ikey_ptr,
GetContext* get_context, BlockCacheLookupContext* lookup_context); GetContext* get_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority);
bool MayMatch(const Slice& slice, bool no_io, const Slice* const_ikey_ptr, bool MayMatch(const Slice& slice, bool no_io, const Slice* const_ikey_ptr,
GetContext* get_context, GetContext* get_context,
BlockCacheLookupContext* lookup_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority,
FilterFunction filter_function) const; FilterFunction filter_function) const;
using FilterManyFunction = void (FullFilterBlockReader::*)( using FilterManyFunction = void (FullFilterBlockReader::*)(
MultiGetRange* range, const SliceTransform* prefix_extractor, MultiGetRange* range, const SliceTransform* prefix_extractor,
const bool no_io, BlockCacheLookupContext* lookup_context); const bool no_io, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority);
void MayMatch(MultiGetRange* range, const SliceTransform* prefix_extractor, void MayMatch(MultiGetRange* range, const SliceTransform* prefix_extractor,
bool no_io, BlockCacheLookupContext* lookup_context, bool no_io, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority,
FilterManyFunction filter_function) const; FilterManyFunction filter_function) const;
void MayMatchPartition(MultiGetRange* range, void MayMatchPartition(MultiGetRange* range,
const SliceTransform* prefix_extractor, const SliceTransform* prefix_extractor,
BlockHandle filter_handle, bool no_io, BlockHandle filter_handle, bool no_io,
BlockCacheLookupContext* lookup_context, BlockCacheLookupContext* lookup_context,
Env::IOPriority rate_limiter_priority,
FilterManyFunction filter_function) const; FilterManyFunction filter_function) const;
Status CacheDependencies(const ReadOptions& ro, bool pin) override; Status CacheDependencies(const ReadOptions& ro, bool pin) override;

23
table/block_based/partitioned_filter_block_test.cc
Unescape View File

@ -164,6 +164,7 @@ class PartitionedFilterBlockTest
PartitionedIndexBuilder* pib, bool empty = false) { PartitionedIndexBuilder* pib, bool empty = false) {
std::unique_ptr<PartitionedFilterBlockReader> reader( std::unique_ptr<PartitionedFilterBlockReader> reader(
NewReader(builder, pib)); NewReader(builder, pib));
Env::IOPriority rate_limiter_priority = Env::IO_TOTAL;
// Querying added keys // Querying added keys
const bool no_io = true; const bool no_io = true;
for (auto key : keys) { for (auto key : keys) {
@ -171,7 +172,8 @@ class PartitionedFilterBlockTest
const Slice ikey_slice = Slice(*ikey.rep()); const Slice ikey_slice = Slice(*ikey.rep());
ASSERT_TRUE(reader->KeyMayMatch(key, !no_io, &ikey_slice, ASSERT_TRUE(reader->KeyMayMatch(key, !no_io, &ikey_slice,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
} }
{ {
// querying a key twice // querying a key twice
@ -179,7 +181,8 @@ class PartitionedFilterBlockTest
const Slice ikey_slice = Slice(*ikey.rep()); const Slice ikey_slice = Slice(*ikey.rep());
ASSERT_TRUE(reader->KeyMayMatch(keys[0], !no_io, &ikey_slice, ASSERT_TRUE(reader->KeyMayMatch(keys[0], !no_io, &ikey_slice,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
} }
// querying missing keys // querying missing keys
for (auto key : missing_keys) { for (auto key : missing_keys) {
@ -188,12 +191,14 @@ class PartitionedFilterBlockTest
if (empty) { if (empty) {
ASSERT_TRUE(reader->KeyMayMatch(key, !no_io, &ikey_slice, ASSERT_TRUE(reader->KeyMayMatch(key, !no_io, &ikey_slice,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
} else { } else {
// assuming a good hash function // assuming a good hash function
ASSERT_FALSE(reader->KeyMayMatch(key, !no_io, &ikey_slice, ASSERT_FALSE(reader->KeyMayMatch(key, !no_io, &ikey_slice,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
} }
} }
} }
@ -345,7 +350,8 @@ TEST_P(PartitionedFilterBlockTest, SamePrefixInMultipleBlocks) {
ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key), ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key),
/*no_io=*/false, &ikey_slice, /*no_io=*/false, &ikey_slice,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
Env::IO_TOTAL));
} }
// Non-existent keys but with the same prefix // Non-existent keys but with the same prefix
const std::string pnonkeys[4] = {"p-key9", "p-key11", "p-key21", "p-key31"}; const std::string pnonkeys[4] = {"p-key9", "p-key11", "p-key21", "p-key31"};
@ -355,7 +361,8 @@ TEST_P(PartitionedFilterBlockTest, SamePrefixInMultipleBlocks) {
ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key), ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key),
/*no_io=*/false, &ikey_slice, /*no_io=*/false, &ikey_slice,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
Env::IO_TOTAL));
} }
} }
@ -386,6 +393,7 @@ TEST_P(PartitionedFilterBlockTest, PrefixInWrongPartitionBug) {
CutABlock(pib.get(), pkeys[4]); CutABlock(pib.get(), pkeys[4]);
std::unique_ptr<PartitionedFilterBlockReader> reader( std::unique_ptr<PartitionedFilterBlockReader> reader(
NewReader(builder.get(), pib.get())); NewReader(builder.get(), pib.get()));
Env::IOPriority rate_limiter_priority = Env::IO_TOTAL;
for (auto key : pkeys) { for (auto key : pkeys) {
auto prefix = prefix_extractor->Transform(key); auto prefix = prefix_extractor->Transform(key);
auto ikey = InternalKey(prefix, 0, ValueType::kTypeValue); auto ikey = InternalKey(prefix, 0, ValueType::kTypeValue);
@ -393,7 +401,8 @@ TEST_P(PartitionedFilterBlockTest, PrefixInWrongPartitionBug) {
ASSERT_TRUE(reader->PrefixMayMatch(prefix, ASSERT_TRUE(reader->PrefixMayMatch(prefix,
/*no_io=*/false, &ikey_slice, /*no_io=*/false, &ikey_slice,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr)); /*lookup_context=*/nullptr,
rate_limiter_priority));
} }
} }

4
util/filter_bench.cc
Unescape View File

@ -20,6 +20,7 @@ int main() {
#include "port/port.h" #include "port/port.h"
#include "port/stack_trace.h" #include "port/stack_trace.h"
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/env.h"
#include "rocksdb/system_clock.h" #include "rocksdb/system_clock.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "table/block_based/filter_policy_internal.h" #include "table/block_based/filter_policy_internal.h"
@ -150,6 +151,7 @@ using ROCKSDB_NAMESPACE::Cache;
using ROCKSDB_NAMESPACE::CacheEntryRole; using ROCKSDB_NAMESPACE::CacheEntryRole;
using ROCKSDB_NAMESPACE::CacheEntryRoleOptions; using ROCKSDB_NAMESPACE::CacheEntryRoleOptions;
using ROCKSDB_NAMESPACE::EncodeFixed32; using ROCKSDB_NAMESPACE::EncodeFixed32;
using ROCKSDB_NAMESPACE::Env;
using ROCKSDB_NAMESPACE::FastRange32; using ROCKSDB_NAMESPACE::FastRange32;
using ROCKSDB_NAMESPACE::FilterBitsReader; using ROCKSDB_NAMESPACE::FilterBitsReader;
using ROCKSDB_NAMESPACE::FilterBuildingContext; using ROCKSDB_NAMESPACE::FilterBuildingContext;
@ -726,7 +728,7 @@ double FilterBench::RandomQueryTest(uint32_t inside_threshold, bool dry_run,
batch_slices[i], batch_slices[i],
/*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
/*get_context=*/nullptr, /*get_context=*/nullptr,
/*lookup_context=*/nullptr); /*lookup_context=*/nullptr, Env::IO_TOTAL);
} }
} else { } else {
if (dry_run) { if (dry_run) {

Write Preview
Loading…
Cancel
Save