diff --git a/CMakeLists.txt b/CMakeLists.txt index 68c28f1b3..16b3cafc8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -602,6 +602,7 @@ set(SOURCES table/block_based/flush_block_policy.cc table/block_based/full_filter_block.cc table/block_based/index_builder.cc + table/block_based/parsed_full_filter_block.cc table/block_based/partitioned_filter_block.cc table/block_based/uncompression_dict_reader.cc table/block_fetcher.cc diff --git a/TARGETS b/TARGETS index e1af516e0..0d175656f 100644 --- a/TARGETS +++ b/TARGETS @@ -233,6 +233,7 @@ cpp_library( "table/block_based/flush_block_policy.cc", "table/block_based/full_filter_block.cc", "table/block_based/index_builder.cc", + "table/block_based/parsed_full_filter_block.cc", "table/block_based/partitioned_filter_block.cc", "table/block_based/uncompression_dict_reader.cc", "table/block_fetcher.cc", diff --git a/src.mk b/src.mk index 42d6ae70b..8cce00207 100644 --- a/src.mk +++ b/src.mk @@ -128,6 +128,7 @@ LIB_SOURCES = \ table/block_based/flush_block_policy.cc \ table/block_based/full_filter_block.cc \ table/block_based/index_builder.cc \ + table/block_based/parsed_full_filter_block.cc \ table/block_based/partitioned_filter_block.cc \ table/block_based/uncompression_dict_reader.cc \ table/block_fetcher.cc \ diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc index 8baf39987..27e1d6cf2 100644 --- a/table/block_based/block_based_table_reader.cc +++ b/table/block_based/block_based_table_reader.cc @@ -84,7 +84,8 @@ class BlocklikeTraits { SequenceNumber /* global_seqno */, size_t /* read_amp_bytes_per_bit */, Statistics* /* statistics */, - bool /* using_zstd */) { + bool /* using_zstd */, + const FilterPolicy* /* filter_policy */) { return new BlockContents(std::move(contents)); } @@ -93,12 +94,30 @@ class BlocklikeTraits { } }; +template <> +class BlocklikeTraits { + public: + static ParsedFullFilterBlock* Create(BlockContents&& contents, + SequenceNumber /* global_seqno */, + size_t /* read_amp_bytes_per_bit */, + Statistics* /* statistics */, + bool /* using_zstd */, + const FilterPolicy* filter_policy) { + return new ParsedFullFilterBlock(filter_policy, std::move(contents)); + } + + static uint32_t GetNumRestarts(const ParsedFullFilterBlock& /* block */) { + return 0; + } +}; + template <> class BlocklikeTraits { public: static Block* Create(BlockContents&& contents, SequenceNumber global_seqno, size_t read_amp_bytes_per_bit, Statistics* statistics, - bool /* using_zstd */) { + bool /* using_zstd */, + const FilterPolicy* /* filter_policy */) { return new Block(std::move(contents), global_seqno, read_amp_bytes_per_bit, statistics); } @@ -115,7 +134,8 @@ class BlocklikeTraits { SequenceNumber /* global_seqno */, size_t /* read_amp_bytes_per_bit */, Statistics* /* statistics */, - bool using_zstd) { + bool using_zstd, + const FilterPolicy* /* filter_policy */) { return new UncompressionDict(contents.data, std::move(contents.allocation), using_zstd); } @@ -141,7 +161,7 @@ Status ReadBlockFromFile( const UncompressionDict& uncompression_dict, const PersistentCacheOptions& cache_options, SequenceNumber global_seqno, size_t read_amp_bytes_per_bit, MemoryAllocator* memory_allocator, - bool for_compaction, bool using_zstd) { + bool for_compaction, bool using_zstd, const FilterPolicy* filter_policy) { assert(result); BlockContents contents; @@ -153,7 +173,7 @@ Status ReadBlockFromFile( if (s.ok()) { result->reset(BlocklikeTraits::Create( std::move(contents), global_seqno, read_amp_bytes_per_bit, - ioptions.statistics, using_zstd)); + ioptions.statistics, using_zstd, filter_policy)); } return s; @@ -1629,7 +1649,7 @@ Status BlockBasedTable::ReadMetaBlock(FilePrefetchBuffer* prefetch_buffer, UncompressionDict::GetEmptyDict(), rep_->persistent_cache_options, kDisableGlobalSequenceNumber, 0 /* read_amp_bytes_per_bit */, GetMemoryAllocator(rep_->table_options), false /* for_compaction */, - rep_->blocks_definitely_zstd_compressed); + rep_->blocks_definitely_zstd_compressed, nullptr /* filter_policy */); if (!s.ok()) { ROCKS_LOG_ERROR(rep_->ioptions.info_log, @@ -1718,7 +1738,8 @@ Status BlockBasedTable::GetDataBlockFromCache( BlocklikeTraits::Create( std::move(contents), rep_->get_global_seqno(block_type), read_amp_bytes_per_bit, statistics, - rep_->blocks_definitely_zstd_compressed)); // uncompressed block + rep_->blocks_definitely_zstd_compressed, + rep_->table_options.filter_policy.get())); // uncompressed block if (block_cache != nullptr && block_holder->own_bytes() && read_options.fill_cache) { @@ -1789,11 +1810,13 @@ Status BlockBasedTable::PutDataBlockToCache( block_holder.reset(BlocklikeTraits::Create( std::move(uncompressed_block_contents), seq_no, read_amp_bytes_per_bit, - statistics, rep_->blocks_definitely_zstd_compressed)); + statistics, rep_->blocks_definitely_zstd_compressed, + rep_->table_options.filter_policy.get())); } else { block_holder.reset(BlocklikeTraits::Create( std::move(*raw_block_contents), seq_no, read_amp_bytes_per_bit, - statistics, rep_->blocks_definitely_zstd_compressed)); + statistics, rep_->blocks_definitely_zstd_compressed, + rep_->table_options.filter_policy.get())); } // Insert compressed block into compressed block cache. @@ -2458,7 +2481,8 @@ Status BlockBasedTable::RetrieveBlock( ? rep_->table_options.read_amp_bytes_per_bit : 0, GetMemoryAllocator(rep_->table_options), for_compaction, - rep_->blocks_definitely_zstd_compressed); + rep_->blocks_definitely_zstd_compressed, + rep_->table_options.filter_policy.get()); } if (!s.ok()) { @@ -2480,6 +2504,13 @@ template Status BlockBasedTable::RetrieveBlock( GetContext* get_context, BlockCacheLookupContext* lookup_context, bool for_compaction, bool use_cache) const; +template Status BlockBasedTable::RetrieveBlock( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, + const BlockHandle& handle, const UncompressionDict& uncompression_dict, + CachableEntry* block_entry, BlockType block_type, + GetContext* get_context, BlockCacheLookupContext* lookup_context, + bool for_compaction, bool use_cache) const; + template Status BlockBasedTable::RetrieveBlock( FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, const BlockHandle& handle, const UncompressionDict& uncompression_dict, diff --git a/table/block_based/filter_block_reader_common.cc b/table/block_based/filter_block_reader_common.cc index b6a334986..49a268823 100644 --- a/table/block_based/filter_block_reader_common.cc +++ b/table/block_based/filter_block_reader_common.cc @@ -7,6 +7,7 @@ #include "table/block_based/filter_block_reader_common.h" #include "monitoring/perf_context_imp.h" #include "table/block_based/block_based_table_reader.h" +#include "table/block_based/parsed_full_filter_block.h" namespace rocksdb { @@ -96,5 +97,6 @@ size_t FilterBlockReaderCommon::ApproximateFilterBlockMemoryUsage() // This makes it possible to keep the template definitions in the .cc file. template class FilterBlockReaderCommon; template class FilterBlockReaderCommon; +template class FilterBlockReaderCommon; } // namespace rocksdb diff --git a/table/block_based/full_filter_block.cc b/table/block_based/full_filter_block.cc index 9a858232d..b3b2f5813 100644 --- a/table/block_based/full_filter_block.cc +++ b/table/block_based/full_filter_block.cc @@ -93,7 +93,8 @@ Slice FullFilterBlockBuilder::Finish(const BlockHandle& /*tmp*/, } FullFilterBlockReader::FullFilterBlockReader( - const BlockBasedTable* t, CachableEntry&& filter_block) + const BlockBasedTable* t, + CachableEntry&& filter_block) : FilterBlockReaderCommon(t, std::move(filter_block)) { const SliceTransform* const prefix_extractor = table_prefix_extractor(); if (prefix_extractor) { @@ -125,7 +126,7 @@ std::unique_ptr FullFilterBlockReader::Create( assert(table->get_rep()); assert(!pin || prefetch); - CachableEntry filter_block; + CachableEntry filter_block; if (prefetch || !use_cache) { const Status s = ReadFilterBlock(table, prefetch_buffer, ReadOptions(), use_cache, nullptr /* get_context */, @@ -158,7 +159,7 @@ bool FullFilterBlockReader::PrefixMayMatch( bool FullFilterBlockReader::MayMatch( const Slice& entry, bool no_io, GetContext* get_context, BlockCacheLookupContext* lookup_context) const { - CachableEntry filter_block; + CachableEntry filter_block; const Status s = GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block); @@ -168,15 +169,10 @@ bool FullFilterBlockReader::MayMatch( assert(filter_block.GetValue()); - if (filter_block.GetValue()->data.size() != 0) { - assert(table()); - assert(table()->get_rep()); - - std::unique_ptr filter_bits_reader( - table()->get_rep()->filter_policy->GetFilterBitsReader( - filter_block.GetValue()->data)); - assert(filter_bits_reader != nullptr); + FilterBitsReader* const filter_bits_reader = + filter_block.GetValue()->filter_bits_reader(); + if (filter_bits_reader) { if (filter_bits_reader->MayMatch(entry)) { PERF_COUNTER_ADD(bloom_sst_hit_count, 1); return true; @@ -220,7 +216,7 @@ void FullFilterBlockReader::PrefixesMayMatch( void FullFilterBlockReader::MayMatch( MultiGetRange* range, bool no_io, const SliceTransform* prefix_extractor, BlockCacheLookupContext* lookup_context) const { - CachableEntry filter_block; + CachableEntry filter_block; const Status s = GetOrReadFilterBlock(no_io, range->begin()->get_context, lookup_context, &filter_block); @@ -230,18 +226,13 @@ void FullFilterBlockReader::MayMatch( assert(filter_block.GetValue()); - if (filter_block.GetValue()->data.size() == 0) { + FilterBitsReader* const filter_bits_reader = + filter_block.GetValue()->filter_bits_reader(); + + if (!filter_bits_reader) { return; } - assert(table()); - assert(table()->get_rep()); - - std::unique_ptr filter_bits_reader( - table()->get_rep()->filter_policy->GetFilterBitsReader( - filter_block.GetValue()->data)); - assert(filter_bits_reader != nullptr); - // We need to use an array instead of autovector for may_match since // &may_match[0] doesn't work for autovector (compiler error). So // declare both keys and may_match as arrays, which is also slightly less @@ -261,6 +252,7 @@ void FullFilterBlockReader::MayMatch( filter_range.SkipKey(iter); } } + filter_bits_reader->MayMatch(num_keys, &keys[0], &may_match[0]); int i = 0; diff --git a/table/block_based/full_filter_block.h b/table/block_based/full_filter_block.h index 65dc278a8..04f1ec228 100644 --- a/table/block_based/full_filter_block.h +++ b/table/block_based/full_filter_block.h @@ -16,7 +16,7 @@ #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" #include "table/block_based/filter_block_reader_common.h" -#include "table/format.h" +#include "table/block_based/parsed_full_filter_block.h" #include "util/hash.h" namespace rocksdb { @@ -80,10 +80,11 @@ class FullFilterBlockBuilder : public FilterBlockBuilder { // A FilterBlockReader is used to parse filter from SST table. // KeyMayMatch and PrefixMayMatch would trigger filter checking -class FullFilterBlockReader : public FilterBlockReaderCommon { +class FullFilterBlockReader + : public FilterBlockReaderCommon { public: FullFilterBlockReader(const BlockBasedTable* t, - CachableEntry&& filter_block); + CachableEntry&& filter_block); static std::unique_ptr Create( const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, diff --git a/table/block_based/full_filter_block_test.cc b/table/block_based/full_filter_block_test.cc index 6ee1092dc..28b2cefaa 100644 --- a/table/block_based/full_filter_block_test.cc +++ b/table/block_based/full_filter_block_test.cc @@ -113,9 +113,10 @@ TEST_F(PluginFullFilterBlockTest, PluginEmptyBuilder) { Slice slice = builder.Finish(); ASSERT_EQ("", EscapeString(slice)); - CachableEntry block( - new BlockContents(slice), nullptr /* cache */, nullptr /* cache_handle */, - true /* own_value */); + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); FullFilterBlockReader reader(table_.get(), std::move(block)); // Remain same symantic with blockbased filter @@ -136,9 +137,10 @@ TEST_F(PluginFullFilterBlockTest, PluginSingleChunk) { builder.Add("hello"); Slice slice = builder.Finish(); - CachableEntry block( - new BlockContents(slice), nullptr /* cache */, nullptr /* cache_handle */, - true /* own_value */); + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); FullFilterBlockReader reader(table_.get(), std::move(block)); ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr, @@ -189,9 +191,10 @@ TEST_F(FullFilterBlockTest, EmptyBuilder) { Slice slice = builder.Finish(); ASSERT_EQ("", EscapeString(slice)); - CachableEntry block( - new BlockContents(slice), nullptr /* cache */, nullptr /* cache_handle */, - true /* own_value */); + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); FullFilterBlockReader reader(table_.get(), std::move(block)); // Remain same symantic with blockbased filter @@ -247,9 +250,10 @@ TEST_F(FullFilterBlockTest, SingleChunk) { ASSERT_EQ(5, builder.NumAdded()); Slice slice = builder.Finish(); - CachableEntry block( - new BlockContents(slice), nullptr /* cache */, nullptr /* cache_handle */, - true /* own_value */); + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); FullFilterBlockReader reader(table_.get(), std::move(block)); ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr, diff --git a/table/block_based/parsed_full_filter_block.cc b/table/block_based/parsed_full_filter_block.cc new file mode 100644 index 000000000..5cc259d19 --- /dev/null +++ b/table/block_based/parsed_full_filter_block.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "table/block_based/parsed_full_filter_block.h" +#include "rocksdb/filter_policy.h" + +namespace rocksdb { + +ParsedFullFilterBlock::ParsedFullFilterBlock(const FilterPolicy* filter_policy, + BlockContents&& contents) + : block_contents_(std::move(contents)), + filter_bits_reader_( + !block_contents_.data.empty() + ? filter_policy->GetFilterBitsReader(block_contents_.data) + : nullptr) {} + +ParsedFullFilterBlock::~ParsedFullFilterBlock() = default; + +} // namespace rocksdb diff --git a/table/block_based/parsed_full_filter_block.h b/table/block_based/parsed_full_filter_block.h new file mode 100644 index 000000000..74f8ccf5a --- /dev/null +++ b/table/block_based/parsed_full_filter_block.h @@ -0,0 +1,40 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "table/format.h" + +namespace rocksdb { + +class FilterBitsReader; +class FilterPolicy; + +// The sharable/cachable part of the full filter. +class ParsedFullFilterBlock { + public: + ParsedFullFilterBlock(const FilterPolicy* filter_policy, + BlockContents&& contents); + ~ParsedFullFilterBlock(); + + FilterBitsReader* filter_bits_reader() const { + return filter_bits_reader_.get(); + } + + // TODO: consider memory usage of FullFilterBitsReader + size_t ApproximateMemoryUsage() const { + return block_contents_.ApproximateMemoryUsage(); + } + + bool own_bytes() const { return block_contents_.own_bytes(); } + + private: + BlockContents block_contents_; + std::unique_ptr filter_bits_reader_; +}; + +} // namespace rocksdb diff --git a/table/block_based/partitioned_filter_block.cc b/table/block_based/partitioned_filter_block.cc index 9ecf56130..b9b96989f 100644 --- a/table/block_based/partitioned_filter_block.cc +++ b/table/block_based/partitioned_filter_block.cc @@ -217,7 +217,7 @@ Status PartitionedFilterBlockReader::GetFilterPartitionBlock( FilePrefetchBuffer* prefetch_buffer, const BlockHandle& fltr_blk_handle, bool no_io, GetContext* get_context, BlockCacheLookupContext* lookup_context, - CachableEntry* filter_block) const { + CachableEntry* filter_block) const { assert(table()); assert(filter_block); assert(filter_block->IsEmpty()); @@ -267,7 +267,7 @@ bool PartitionedFilterBlockReader::MayMatch( return false; } - CachableEntry filter_partition_block; + CachableEntry filter_partition_block; s = GetFilterPartitionBlock(nullptr /* prefetch_buffer */, filter_handle, no_io, get_context, lookup_context, &filter_partition_block); @@ -346,7 +346,7 @@ void PartitionedFilterBlockReader::CacheDependencies(bool pin) { for (biter.SeekToFirst(); biter.Valid(); biter.Next()) { handle = biter.value().handle; - CachableEntry block; + CachableEntry block; // TODO: Support counter batch update for partitioned index and // filter blocks s = table()->MaybeReadBlockAndLoadToCache( diff --git a/table/block_based/partitioned_filter_block.h b/table/block_based/partitioned_filter_block.h index 9cac1b88a..089773d47 100644 --- a/table/block_based/partitioned_filter_block.h +++ b/table/block_based/partitioned_filter_block.h @@ -100,7 +100,7 @@ class PartitionedFilterBlockReader : public FilterBlockReaderCommon { FilePrefetchBuffer* prefetch_buffer, const BlockHandle& handle, bool no_io, GetContext* get_context, BlockCacheLookupContext* lookup_context, - CachableEntry* filter_block) const; + CachableEntry* filter_block) const; using FilterFunction = bool (FullFilterBlockReader::*)( const Slice& slice, const SliceTransform* prefix_extractor, @@ -119,7 +119,8 @@ class PartitionedFilterBlockReader : public FilterBlockReaderCommon { bool index_value_is_full() const; protected: - std::unordered_map> filter_map_; + std::unordered_map> + filter_map_; }; } // namespace rocksdb diff --git a/table/block_based/partitioned_filter_block_test.cc b/table/block_based/partitioned_filter_block_test.cc index f849d62ed..ee93262ad 100644 --- a/table/block_based/partitioned_filter_block_test.cc +++ b/table/block_based/partitioned_filter_block_test.cc @@ -20,7 +20,7 @@ namespace rocksdb { -std::map slices; +std::map blooms; class MockedBlockBasedTable : public BlockBasedTable { public: @@ -37,13 +37,18 @@ class MyPartitionedFilterBlockReader : public PartitionedFilterBlockReader { MyPartitionedFilterBlockReader(BlockBasedTable* t, CachableEntry&& filter_block) : PartitionedFilterBlockReader(t, std::move(filter_block)) { - for (const auto& pair : slices) { + for (const auto& pair : blooms) { const uint64_t offset = pair.first; - const Slice& slice = pair.second; - - CachableEntry block( - new BlockContents(slice), nullptr /* cache */, - nullptr /* cache_handle */, true /* own_value */); + const std::string& bloom = pair.second; + + assert(t); + assert(t->get_rep()); + CachableEntry block( + new ParsedFullFilterBlock( + t->get_rep()->table_options.filter_policy.get(), + BlockContents(Slice(bloom))), + nullptr /* cache */, nullptr /* cache_handle */, + true /* own_value */); filter_map_[offset] = std::move(block); } } @@ -101,7 +106,7 @@ class PartitionedFilterBlockTest uint64_t last_offset = 10; BlockHandle Write(const Slice& slice) { BlockHandle bh(last_offset + 1, slice.size()); - slices[bh.offset()] = slice; + blooms[bh.offset()] = slice.ToString(); last_offset += bh.size(); return bh; } diff --git a/util/filter_bench.cc b/util/filter_bench.cc index c12bef669..359292e99 100644 --- a/util/filter_bench.cc +++ b/util/filter_bench.cc @@ -70,6 +70,7 @@ using rocksdb::fastrange32; using rocksdb::FilterBitsBuilder; using rocksdb::FilterBitsReader; using rocksdb::FullFilterBlockReader; +using rocksdb::ParsedFullFilterBlock; using rocksdb::Random32; using rocksdb::Slice; using rocksdb::mock::MockBlockBasedTableTester; @@ -208,9 +209,10 @@ void FilterBench::Go() { info.keys_added_ = keys_to_add; info.reader_.reset( table_options_.filter_policy->GetFilterBitsReader(info.filter_)); - CachableEntry block( - new BlockContents(info.filter_), nullptr /* cache */, - nullptr /* cache_handle */, true /* own_value */); + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(info.filter_)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); info.full_block_reader_.reset( new FullFilterBlockReader(table_.get(), std::move(block))); total_memory_used += info.filter_.size();