Summary:
1. Make filter_block.h a base class. Derive block_based_filter_block and full_filter_block. The previous one is the traditional filter block. The full_filter_block is newly added. It would generate a filter block that contain all the keys in SST file.
2. When querying a key, table would first check if full_filter is available. If not, it would go to the exact data block and check using block_based filter.
3. User could choose to use full_filter or tradional(block_based_filter). They would be stored in SST file with different meta index name. "filter.filter_policy" or "full_filter.filter_policy". Then, Table reader is able to know the fllter block type.
4. Some optimizations have been done for full_filter_block, thus it requires a different interface compared to the original one in filter_policy.h.
5. Actual implementation of filter bits coding/decoding is placed in util/bloom_impl.cc
Benchmark: base commit 1d23b5c470
Command:
db_bench --db=/dev/shm/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --write_buffer_size=134217728 --max_write_buffer_number=2 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --verify_checksum=false --max_background_compactions=4 --use_plain_table=0 --memtablerep=prefix_hash --open_files=-1 --mmap_read=1 --mmap_write=0 --bloom_bits=10 --bloom_locality=1 --memtable_bloom_bits=500000 --compression_type=lz4 --num=393216000 --use_hash_search=1 --block_size=1024 --block_restart_interval=16 --use_existing_db=1 --threads=1 --benchmarks=readrandom —disable_auto_compactions=1
Read QPS increase for about 30% from 2230002 to 2991411.
Test Plan:
make all check
valgrind db_test
db_stress --use_block_based_filter = 0
./auto_sanity_test.sh
Reviewers: igor, yhchiang, ljin, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D20979
main
Feng Zhu
10 years ago
parent
9360cc690e
commit
0af157f9bf
@ -0,0 +1,102 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
//
|
||||||
|
// A filter block is stored near the end of a Table file. It contains
|
||||||
|
// filters (e.g., bloom filters) for all data blocks in the table combined
|
||||||
|
// into a single filter block.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stddef.h> |
||||||
|
#include <stdint.h> |
||||||
|
#include <string> |
||||||
|
#include <memory> |
||||||
|
#include <vector> |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/slice_transform.h" |
||||||
|
#include "table/filter_block.h" |
||||||
|
#include "util/hash.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
|
||||||
|
// A BlockBasedFilterBlockBuilder is used to construct all of the filters for a
|
||||||
|
// particular Table. It generates a single string which is stored as
|
||||||
|
// a special block in the Table.
|
||||||
|
//
|
||||||
|
// The sequence of calls to BlockBasedFilterBlockBuilder must match the regexp:
|
||||||
|
// (StartBlock Add*)* Finish
|
||||||
|
class BlockBasedFilterBlockBuilder : public FilterBlockBuilder { |
||||||
|
public: |
||||||
|
BlockBasedFilterBlockBuilder(const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt); |
||||||
|
|
||||||
|
virtual bool IsBlockBased() override { return true; } |
||||||
|
virtual void StartBlock(uint64_t block_offset) override; |
||||||
|
virtual void Add(const Slice& key) override; |
||||||
|
virtual Slice Finish() override; |
||||||
|
|
||||||
|
private: |
||||||
|
void AddKey(const Slice& key); |
||||||
|
void AddPrefix(const Slice& key); |
||||||
|
void GenerateFilter(); |
||||||
|
|
||||||
|
// important: all of these might point to invalid addresses
|
||||||
|
// at the time of destruction of this filter block. destructor
|
||||||
|
// should NOT dereference them.
|
||||||
|
const FilterPolicy* policy_; |
||||||
|
const SliceTransform* prefix_extractor_; |
||||||
|
bool whole_key_filtering_; |
||||||
|
|
||||||
|
std::string entries_; // Flattened entry contents
|
||||||
|
std::vector<size_t> start_; // Starting index in entries_ of each entry
|
||||||
|
uint32_t added_to_start_; // To indicate if key is added
|
||||||
|
std::string result_; // Filter data computed so far
|
||||||
|
std::vector<Slice> tmp_entries_; // policy_->CreateFilter() argument
|
||||||
|
std::vector<uint32_t> filter_offsets_; |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
BlockBasedFilterBlockBuilder(const BlockBasedFilterBlockBuilder&); |
||||||
|
void operator=(const BlockBasedFilterBlockBuilder&); |
||||||
|
}; |
||||||
|
|
||||||
|
// A FilterBlockReader is used to parse filter from SST table.
|
||||||
|
// KeyMayMatch and PrefixMayMatch would trigger filter checking
|
||||||
|
class BlockBasedFilterBlockReader : public FilterBlockReader { |
||||||
|
public: |
||||||
|
// REQUIRES: "contents" and *policy must stay live while *this is live.
|
||||||
|
BlockBasedFilterBlockReader(const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt, |
||||||
|
const Slice& contents, |
||||||
|
bool delete_contents_after_use = false); |
||||||
|
virtual bool IsBlockBased() override { return true; } |
||||||
|
virtual bool KeyMayMatch(const Slice& key, |
||||||
|
uint64_t block_offset = kNotValid) override; |
||||||
|
virtual bool PrefixMayMatch(const Slice& prefix, |
||||||
|
uint64_t block_offset = kNotValid) override; |
||||||
|
virtual size_t ApproximateMemoryUsage() const override; |
||||||
|
|
||||||
|
private: |
||||||
|
const FilterPolicy* policy_; |
||||||
|
const SliceTransform* prefix_extractor_; |
||||||
|
bool whole_key_filtering_; |
||||||
|
const char* data_; // Pointer to filter data (at block-start)
|
||||||
|
const char* offset_; // Pointer to beginning of offset array (at block-end)
|
||||||
|
size_t num_; // Number of entries in offset array
|
||||||
|
size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file)
|
||||||
|
std::unique_ptr<const char[]> filter_data; |
||||||
|
|
||||||
|
bool MayMatch(const Slice& entry, uint64_t block_offset); |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
BlockBasedFilterBlockReader(const BlockBasedFilterBlockReader&); |
||||||
|
void operator=(const BlockBasedFilterBlockReader&); |
||||||
|
}; |
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,242 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "table/block_based_filter_block.h" |
||||||
|
|
||||||
|
#include "rocksdb/filter_policy.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/hash.h" |
||||||
|
#include "util/logging.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
// For testing: emit an array with one hash value per key
|
||||||
|
class TestHashFilter : public FilterPolicy { |
||||||
|
public: |
||||||
|
virtual const char* Name() const { |
||||||
|
return "TestHashFilter"; |
||||||
|
} |
||||||
|
|
||||||
|
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
||||||
|
for (int i = 0; i < n; i++) { |
||||||
|
uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); |
||||||
|
PutFixed32(dst, h); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { |
||||||
|
uint32_t h = Hash(key.data(), key.size(), 1); |
||||||
|
for (unsigned int i = 0; i + 4 <= filter.size(); i += 4) { |
||||||
|
if (h == DecodeFixed32(filter.data() + i)) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
class FilterBlockTest { |
||||||
|
public: |
||||||
|
TestHashFilter policy_; |
||||||
|
BlockBasedTableOptions table_options_; |
||||||
|
|
||||||
|
FilterBlockTest() { |
||||||
|
table_options_.filter_policy.reset(new TestHashFilter()); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(FilterBlockTest, EmptyBuilder) { |
||||||
|
BlockBasedFilterBlockBuilder builder(nullptr, table_options_); |
||||||
|
Slice block = builder.Finish(); |
||||||
|
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); |
||||||
|
BlockBasedFilterBlockReader reader(nullptr, table_options_, block); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 100000)); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(FilterBlockTest, SingleChunk) { |
||||||
|
BlockBasedFilterBlockBuilder builder(nullptr, table_options_); |
||||||
|
builder.StartBlock(100); |
||||||
|
builder.Add("foo"); |
||||||
|
builder.Add("bar"); |
||||||
|
builder.Add("box"); |
||||||
|
builder.StartBlock(200); |
||||||
|
builder.Add("box"); |
||||||
|
builder.StartBlock(300); |
||||||
|
builder.Add("hello"); |
||||||
|
Slice block = builder.Finish(); |
||||||
|
BlockBasedFilterBlockReader reader(nullptr, table_options_, block); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("bar", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("hello", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("missing", 100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("other", 100)); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(FilterBlockTest, MultiChunk) { |
||||||
|
BlockBasedFilterBlockBuilder builder(nullptr, table_options_); |
||||||
|
|
||||||
|
// First filter
|
||||||
|
builder.StartBlock(0); |
||||||
|
builder.Add("foo"); |
||||||
|
builder.StartBlock(2000); |
||||||
|
builder.Add("bar"); |
||||||
|
|
||||||
|
// Second filter
|
||||||
|
builder.StartBlock(3100); |
||||||
|
builder.Add("box"); |
||||||
|
|
||||||
|
// Third filter is empty
|
||||||
|
|
||||||
|
// Last filter
|
||||||
|
builder.StartBlock(9000); |
||||||
|
builder.Add("box"); |
||||||
|
builder.Add("hello"); |
||||||
|
|
||||||
|
Slice block = builder.Finish(); |
||||||
|
BlockBasedFilterBlockReader reader(nullptr, table_options_, block); |
||||||
|
|
||||||
|
// Check first filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("bar", 2000)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("box", 0)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("hello", 0)); |
||||||
|
|
||||||
|
// Check second filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box", 3100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("foo", 3100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("bar", 3100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("hello", 3100)); |
||||||
|
|
||||||
|
// Check third filter (empty)
|
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("foo", 4100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("bar", 4100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("box", 4100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("hello", 4100)); |
||||||
|
|
||||||
|
// Check last filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box", 9000)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("hello", 9000)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("foo", 9000)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("bar", 9000)); |
||||||
|
} |
||||||
|
|
||||||
|
// Test for block based filter block
|
||||||
|
// use new interface in FilterPolicy to create filter builder/reader
|
||||||
|
class BlockBasedFilterBlockTest { |
||||||
|
public: |
||||||
|
BlockBasedTableOptions table_options_; |
||||||
|
|
||||||
|
BlockBasedFilterBlockTest() { |
||||||
|
table_options_.filter_policy.reset(NewBloomFilterPolicy(10)); |
||||||
|
} |
||||||
|
|
||||||
|
~BlockBasedFilterBlockTest() {} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(BlockBasedFilterBlockTest, BlockBasedEmptyBuilder) { |
||||||
|
FilterBlockBuilder* builder = new BlockBasedFilterBlockBuilder( |
||||||
|
nullptr, table_options_); |
||||||
|
Slice block = builder->Finish(); |
||||||
|
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); |
||||||
|
FilterBlockReader* reader = new BlockBasedFilterBlockReader( |
||||||
|
nullptr, table_options_, block); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 100000)); |
||||||
|
|
||||||
|
delete builder; |
||||||
|
delete reader; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(BlockBasedFilterBlockTest, BlockBasedSingleChunk) { |
||||||
|
FilterBlockBuilder* builder = new BlockBasedFilterBlockBuilder( |
||||||
|
nullptr, table_options_); |
||||||
|
builder->StartBlock(100); |
||||||
|
builder->Add("foo"); |
||||||
|
builder->Add("bar"); |
||||||
|
builder->Add("box"); |
||||||
|
builder->StartBlock(200); |
||||||
|
builder->Add("box"); |
||||||
|
builder->StartBlock(300); |
||||||
|
builder->Add("hello"); |
||||||
|
Slice block = builder->Finish(); |
||||||
|
FilterBlockReader* reader = new BlockBasedFilterBlockReader( |
||||||
|
nullptr, table_options_, block); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("bar", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("box", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("hello", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("missing", 100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("other", 100)); |
||||||
|
|
||||||
|
delete builder; |
||||||
|
delete reader; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(BlockBasedFilterBlockTest, BlockBasedMultiChunk) { |
||||||
|
FilterBlockBuilder* builder = new BlockBasedFilterBlockBuilder( |
||||||
|
nullptr, table_options_); |
||||||
|
|
||||||
|
// First filter
|
||||||
|
builder->StartBlock(0); |
||||||
|
builder->Add("foo"); |
||||||
|
builder->StartBlock(2000); |
||||||
|
builder->Add("bar"); |
||||||
|
|
||||||
|
// Second filter
|
||||||
|
builder->StartBlock(3100); |
||||||
|
builder->Add("box"); |
||||||
|
|
||||||
|
// Third filter is empty
|
||||||
|
|
||||||
|
// Last filter
|
||||||
|
builder->StartBlock(9000); |
||||||
|
builder->Add("box"); |
||||||
|
builder->Add("hello"); |
||||||
|
|
||||||
|
Slice block = builder->Finish(); |
||||||
|
FilterBlockReader* reader = new BlockBasedFilterBlockReader( |
||||||
|
nullptr, table_options_, block); |
||||||
|
|
||||||
|
// Check first filter
|
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("bar", 2000)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("box", 0)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("hello", 0)); |
||||||
|
|
||||||
|
// Check second filter
|
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("box", 3100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("foo", 3100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("bar", 3100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("hello", 3100)); |
||||||
|
|
||||||
|
// Check third filter (empty)
|
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("foo", 4100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("bar", 4100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("box", 4100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("hello", 4100)); |
||||||
|
|
||||||
|
// Check last filter
|
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("box", 9000)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("hello", 9000)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("foo", 9000)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("bar", 9000)); |
||||||
|
|
||||||
|
delete builder; |
||||||
|
delete reader; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||||
@ -1,139 +0,0 @@ |
|||||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under the BSD-style license found in the
|
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
|
||||||
//
|
|
||||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a BSD-style license that can be
|
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
||||||
|
|
||||||
#include "table/filter_block.h" |
|
||||||
|
|
||||||
#include "rocksdb/filter_policy.h" |
|
||||||
#include "util/coding.h" |
|
||||||
#include "util/hash.h" |
|
||||||
#include "util/logging.h" |
|
||||||
#include "util/testharness.h" |
|
||||||
#include "util/testutil.h" |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
// For testing: emit an array with one hash value per key
|
|
||||||
class TestHashFilter : public FilterPolicy { |
|
||||||
public: |
|
||||||
virtual const char* Name() const { |
|
||||||
return "TestHashFilter"; |
|
||||||
} |
|
||||||
|
|
||||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
|
||||||
for (int i = 0; i < n; i++) { |
|
||||||
uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); |
|
||||||
PutFixed32(dst, h); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { |
|
||||||
uint32_t h = Hash(key.data(), key.size(), 1); |
|
||||||
for (unsigned int i = 0; i + 4 <= filter.size(); i += 4) { |
|
||||||
if (h == DecodeFixed32(filter.data() + i)) { |
|
||||||
return true; |
|
||||||
} |
|
||||||
} |
|
||||||
return false; |
|
||||||
} |
|
||||||
}; |
|
||||||
|
|
||||||
class FilterBlockTest { |
|
||||||
public: |
|
||||||
const Comparator* comparator_; |
|
||||||
BlockBasedTableOptions table_options_; |
|
||||||
|
|
||||||
FilterBlockTest() |
|
||||||
: comparator_(BytewiseComparator()) { |
|
||||||
table_options_.filter_policy.reset(new TestHashFilter()); |
|
||||||
} |
|
||||||
}; |
|
||||||
|
|
||||||
TEST(FilterBlockTest, EmptyBuilder) { |
|
||||||
FilterBlockBuilder builder(nullptr, table_options_, comparator_); |
|
||||||
Slice block = builder.Finish(); |
|
||||||
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); |
|
||||||
FilterBlockReader reader(nullptr, table_options_, block); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); |
|
||||||
} |
|
||||||
|
|
||||||
TEST(FilterBlockTest, SingleChunk) { |
|
||||||
FilterBlockBuilder builder(nullptr, table_options_, comparator_); |
|
||||||
builder.StartBlock(100); |
|
||||||
builder.AddKey("foo"); |
|
||||||
builder.AddKey("bar"); |
|
||||||
builder.AddKey("box"); |
|
||||||
builder.StartBlock(200); |
|
||||||
builder.AddKey("box"); |
|
||||||
builder.StartBlock(300); |
|
||||||
builder.AddKey("hello"); |
|
||||||
Slice block = builder.Finish(); |
|
||||||
FilterBlockReader reader(nullptr, table_options_, block); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(100, "bar")); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(100, "box")); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(100, "hello")); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(100, "missing")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(100, "other")); |
|
||||||
} |
|
||||||
|
|
||||||
TEST(FilterBlockTest, MultiChunk) { |
|
||||||
FilterBlockBuilder builder(nullptr, table_options_, comparator_); |
|
||||||
|
|
||||||
// First filter
|
|
||||||
builder.StartBlock(0); |
|
||||||
builder.AddKey("foo"); |
|
||||||
builder.StartBlock(2000); |
|
||||||
builder.AddKey("bar"); |
|
||||||
|
|
||||||
// Second filter
|
|
||||||
builder.StartBlock(3100); |
|
||||||
builder.AddKey("box"); |
|
||||||
|
|
||||||
// Third filter is empty
|
|
||||||
|
|
||||||
// Last filter
|
|
||||||
builder.StartBlock(9000); |
|
||||||
builder.AddKey("box"); |
|
||||||
builder.AddKey("hello"); |
|
||||||
|
|
||||||
Slice block = builder.Finish(); |
|
||||||
FilterBlockReader reader(nullptr, table_options_, block); |
|
||||||
|
|
||||||
// Check first filter
|
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(2000, "bar")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(0, "box")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(0, "hello")); |
|
||||||
|
|
||||||
// Check second filter
|
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(3100, "box")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "foo")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "bar")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "hello")); |
|
||||||
|
|
||||||
// Check third filter (empty)
|
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "foo")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "bar")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "box")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "hello")); |
|
||||||
|
|
||||||
// Check last filter
|
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(9000, "box")); |
|
||||||
ASSERT_TRUE(reader.KeyMayMatch(9000, "hello")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(9000, "foo")); |
|
||||||
ASSERT_TRUE(! reader.KeyMayMatch(9000, "bar")); |
|
||||||
} |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
|
|
||||||
int main(int argc, char** argv) { |
|
||||||
return rocksdb::test::RunAllTests(); |
|
||||||
} |
|
||||||
@ -0,0 +1,99 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "table/full_filter_block.h" |
||||||
|
|
||||||
|
#include "rocksdb/filter_policy.h" |
||||||
|
#include "port/port.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
FullFilterBlockBuilder::FullFilterBlockBuilder( |
||||||
|
const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt, |
||||||
|
FilterBitsBuilder* filter_bits_builder) |
||||||
|
: prefix_extractor_(prefix_extractor), |
||||||
|
whole_key_filtering_(table_opt.whole_key_filtering), |
||||||
|
num_added_(0) { |
||||||
|
assert(filter_bits_builder != nullptr); |
||||||
|
filter_bits_builder_.reset(filter_bits_builder); |
||||||
|
} |
||||||
|
|
||||||
|
void FullFilterBlockBuilder::Add(const Slice& key) { |
||||||
|
if (whole_key_filtering_) { |
||||||
|
AddKey(key); |
||||||
|
} |
||||||
|
if (prefix_extractor_ && prefix_extractor_->InDomain(key)) { |
||||||
|
AddPrefix(key); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Add key to filter if needed
|
||||||
|
inline void FullFilterBlockBuilder::AddKey(const Slice& key) { |
||||||
|
filter_bits_builder_->AddKey(key); |
||||||
|
num_added_++; |
||||||
|
} |
||||||
|
|
||||||
|
// Add prefix to filter if needed
|
||||||
|
inline void FullFilterBlockBuilder::AddPrefix(const Slice& key) { |
||||||
|
Slice prefix = prefix_extractor_->Transform(key); |
||||||
|
filter_bits_builder_->AddKey(prefix); |
||||||
|
num_added_++; |
||||||
|
} |
||||||
|
|
||||||
|
Slice FullFilterBlockBuilder::Finish() { |
||||||
|
if (num_added_ != 0) { |
||||||
|
num_added_ = 0; |
||||||
|
return filter_bits_builder_->Finish(&filter_data); |
||||||
|
} |
||||||
|
return Slice(); |
||||||
|
} |
||||||
|
|
||||||
|
FullFilterBlockReader::FullFilterBlockReader( |
||||||
|
const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt, |
||||||
|
const Slice& contents, |
||||||
|
FilterBitsReader* filter_bits_reader, bool delete_contents_after_use) |
||||||
|
: prefix_extractor_(prefix_extractor), |
||||||
|
whole_key_filtering_(table_opt.whole_key_filtering), |
||||||
|
contents_(contents) { |
||||||
|
assert(filter_bits_reader != nullptr); |
||||||
|
filter_bits_reader_.reset(filter_bits_reader); |
||||||
|
|
||||||
|
if (delete_contents_after_use) { |
||||||
|
filter_data.reset(contents.data()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
bool FullFilterBlockReader::KeyMayMatch(const Slice& key, |
||||||
|
uint64_t block_offset) { |
||||||
|
assert(block_offset == kNotValid); |
||||||
|
if (!whole_key_filtering_) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
return MayMatch(key); |
||||||
|
} |
||||||
|
|
||||||
|
bool FullFilterBlockReader::PrefixMayMatch(const Slice& prefix, |
||||||
|
uint64_t block_offset) { |
||||||
|
assert(block_offset == kNotValid); |
||||||
|
if (!prefix_extractor_) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
return MayMatch(prefix); |
||||||
|
} |
||||||
|
|
||||||
|
bool FullFilterBlockReader::MayMatch(const Slice& entry) { |
||||||
|
if (contents_.size() != 0) { |
||||||
|
return filter_bits_reader_->MayMatch(entry); |
||||||
|
} |
||||||
|
return true; // remain the same with block_based filter
|
||||||
|
} |
||||||
|
|
||||||
|
size_t FullFilterBlockReader::ApproximateMemoryUsage() const { |
||||||
|
return contents_.size(); |
||||||
|
} |
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,107 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stddef.h> |
||||||
|
#include <stdint.h> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/slice_transform.h" |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "util/hash.h" |
||||||
|
#include "table/filter_block.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class FilterPolicy; |
||||||
|
class FilterBitsBuilder; |
||||||
|
class FilterBitsReader; |
||||||
|
|
||||||
|
// A FullFilterBlockBuilder is used to construct a full filter for a
|
||||||
|
// particular Table. It generates a single string which is stored as
|
||||||
|
// a special block in the Table.
|
||||||
|
// The format of full filter block is:
|
||||||
|
// +----------------------------------------------------------------+
|
||||||
|
// | full filter for all keys in sst file |
|
||||||
|
// +----------------------------------------------------------------+
|
||||||
|
// The full filter can be very large. At the end of it, we put
|
||||||
|
// num_probes: how many hash functions are used in bloom filter
|
||||||
|
//
|
||||||
|
class FullFilterBlockBuilder : public FilterBlockBuilder { |
||||||
|
public: |
||||||
|
explicit FullFilterBlockBuilder(const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt, |
||||||
|
FilterBitsBuilder* filter_bits_builder); |
||||||
|
// bits_builder is created in filter_policy, it should be passed in here
|
||||||
|
// directly. and be deleted here
|
||||||
|
~FullFilterBlockBuilder() {} |
||||||
|
|
||||||
|
virtual bool IsBlockBased() override { return false; } |
||||||
|
virtual void StartBlock(uint64_t block_offset) override {} |
||||||
|
virtual void Add(const Slice& key) override; |
||||||
|
virtual Slice Finish() override; |
||||||
|
|
||||||
|
private: |
||||||
|
// important: all of these might point to invalid addresses
|
||||||
|
// at the time of destruction of this filter block. destructor
|
||||||
|
// should NOT dereference them.
|
||||||
|
const SliceTransform* prefix_extractor_; |
||||||
|
bool whole_key_filtering_; |
||||||
|
|
||||||
|
uint32_t num_added_; |
||||||
|
std::unique_ptr<FilterBitsBuilder> filter_bits_builder_; |
||||||
|
std::unique_ptr<const char[]> filter_data; |
||||||
|
|
||||||
|
void AddKey(const Slice& key); |
||||||
|
void AddPrefix(const Slice& key); |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
FullFilterBlockBuilder(const FullFilterBlockBuilder&); |
||||||
|
void operator=(const FullFilterBlockBuilder&); |
||||||
|
}; |
||||||
|
|
||||||
|
// A FilterBlockReader is used to parse filter from SST table.
|
||||||
|
// KeyMayMatch and PrefixMayMatch would trigger filter checking
|
||||||
|
class FullFilterBlockReader : public FilterBlockReader { |
||||||
|
public: |
||||||
|
// REQUIRES: "contents" and filter_bits_reader must stay live
|
||||||
|
// while *this is live.
|
||||||
|
explicit FullFilterBlockReader(const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt, |
||||||
|
const Slice& contents, |
||||||
|
FilterBitsReader* filter_bits_reader, |
||||||
|
bool delete_contents_after_use = false); |
||||||
|
|
||||||
|
// bits_reader is created in filter_policy, it should be passed in here
|
||||||
|
// directly. and be deleted here
|
||||||
|
~FullFilterBlockReader() {} |
||||||
|
|
||||||
|
virtual bool IsBlockBased() override { return false; } |
||||||
|
virtual bool KeyMayMatch(const Slice& key, |
||||||
|
uint64_t block_offset = kNotValid) override; |
||||||
|
virtual bool PrefixMayMatch(const Slice& prefix, |
||||||
|
uint64_t block_offset = kNotValid) override; |
||||||
|
virtual size_t ApproximateMemoryUsage() const override; |
||||||
|
|
||||||
|
private: |
||||||
|
const SliceTransform* prefix_extractor_; |
||||||
|
bool whole_key_filtering_; |
||||||
|
|
||||||
|
std::unique_ptr<FilterBitsReader> filter_bits_reader_; |
||||||
|
Slice contents_; |
||||||
|
std::unique_ptr<const char[]> filter_data; |
||||||
|
|
||||||
|
bool MayMatch(const Slice& entry); |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
FullFilterBlockReader(const FullFilterBlockReader&); |
||||||
|
void operator=(const FullFilterBlockReader&); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,181 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "table/full_filter_block.h" |
||||||
|
|
||||||
|
#include "rocksdb/filter_policy.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/hash.h" |
||||||
|
#include "util/logging.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class TestFilterBitsBuilder : public FilterBitsBuilder { |
||||||
|
public: |
||||||
|
explicit TestFilterBitsBuilder() {} |
||||||
|
|
||||||
|
// Add Key to filter
|
||||||
|
virtual void AddKey(const Slice& key) override { |
||||||
|
hash_entries_.push_back(Hash(key.data(), key.size(), 1)); |
||||||
|
} |
||||||
|
|
||||||
|
// Generate the filter using the keys that are added
|
||||||
|
virtual Slice Finish(std::unique_ptr<const char[]>* buf) override { |
||||||
|
uint32_t len = hash_entries_.size() * 4; |
||||||
|
char* data = new char[len]; |
||||||
|
for (size_t i = 0; i < hash_entries_.size(); i++) { |
||||||
|
EncodeFixed32(data + i * 4, hash_entries_[i]); |
||||||
|
} |
||||||
|
buf->reset(data); |
||||||
|
return Slice(data, len); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
std::vector<uint32_t> hash_entries_; |
||||||
|
}; |
||||||
|
|
||||||
|
class TestFilterBitsReader : public FilterBitsReader { |
||||||
|
public: |
||||||
|
explicit TestFilterBitsReader(const Slice& contents) |
||||||
|
: data_(contents.data()), len_(contents.size()) {} |
||||||
|
|
||||||
|
virtual bool MayMatch(const Slice& entry) override { |
||||||
|
uint32_t h = Hash(entry.data(), entry.size(), 1); |
||||||
|
for (size_t i = 0; i + 4 <= len_; i += 4) { |
||||||
|
if (h == DecodeFixed32(data_ + i)) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
const char* data_; |
||||||
|
uint32_t len_; |
||||||
|
}; |
||||||
|
|
||||||
|
|
||||||
|
class TestHashFilter : public FilterPolicy { |
||||||
|
public: |
||||||
|
virtual const char* Name() const { |
||||||
|
return "TestHashFilter"; |
||||||
|
} |
||||||
|
|
||||||
|
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
||||||
|
for (int i = 0; i < n; i++) { |
||||||
|
uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); |
||||||
|
PutFixed32(dst, h); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { |
||||||
|
uint32_t h = Hash(key.data(), key.size(), 1); |
||||||
|
for (unsigned int i = 0; i + 4 <= filter.size(); i += 4) { |
||||||
|
if (h == DecodeFixed32(filter.data() + i)) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
virtual FilterBitsBuilder* GetFilterBitsBuilder() const override { |
||||||
|
return new TestFilterBitsBuilder(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual FilterBitsReader* GetFilterBitsReader(const Slice& contents) |
||||||
|
const override { |
||||||
|
return new TestFilterBitsReader(contents); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
class PluginFullFilterBlockTest { |
||||||
|
public: |
||||||
|
BlockBasedTableOptions table_options_; |
||||||
|
|
||||||
|
PluginFullFilterBlockTest() { |
||||||
|
table_options_.filter_policy.reset(new TestHashFilter()); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(PluginFullFilterBlockTest, PluginEmptyBuilder) { |
||||||
|
FullFilterBlockBuilder builder(nullptr, table_options_, |
||||||
|
table_options_.filter_policy->GetFilterBitsBuilder()); |
||||||
|
Slice block = builder.Finish(); |
||||||
|
ASSERT_EQ("", EscapeString(block)); |
||||||
|
|
||||||
|
FullFilterBlockReader reader(nullptr, table_options_, block, |
||||||
|
table_options_.filter_policy->GetFilterBitsReader(block)); |
||||||
|
// Remain same symantic with blockbased filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo")); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(PluginFullFilterBlockTest, PluginSingleChunk) { |
||||||
|
FullFilterBlockBuilder builder(nullptr, table_options_, |
||||||
|
table_options_.filter_policy->GetFilterBitsBuilder()); |
||||||
|
builder.Add("foo"); |
||||||
|
builder.Add("bar"); |
||||||
|
builder.Add("box"); |
||||||
|
builder.Add("box"); |
||||||
|
builder.Add("hello"); |
||||||
|
Slice block = builder.Finish(); |
||||||
|
FullFilterBlockReader reader(nullptr, table_options_, block, |
||||||
|
table_options_.filter_policy->GetFilterBitsReader(block)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("bar")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("hello")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo")); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("missing")); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("other")); |
||||||
|
} |
||||||
|
|
||||||
|
class FullFilterBlockTest { |
||||||
|
public: |
||||||
|
BlockBasedTableOptions table_options_; |
||||||
|
|
||||||
|
FullFilterBlockTest() { |
||||||
|
table_options_.filter_policy.reset(NewBloomFilterPolicy(10, false)); |
||||||
|
} |
||||||
|
|
||||||
|
~FullFilterBlockTest() {} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(FullFilterBlockTest, EmptyBuilder) { |
||||||
|
FullFilterBlockBuilder builder(nullptr, table_options_, |
||||||
|
table_options_.filter_policy->GetFilterBitsBuilder()); |
||||||
|
Slice block = builder.Finish(); |
||||||
|
ASSERT_EQ("", EscapeString(block)); |
||||||
|
|
||||||
|
FullFilterBlockReader reader(nullptr, table_options_, block, |
||||||
|
table_options_.filter_policy->GetFilterBitsReader(block)); |
||||||
|
// Remain same symantic with blockbased filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo")); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(FullFilterBlockTest, SingleChunk) { |
||||||
|
FullFilterBlockBuilder builder(nullptr, table_options_, |
||||||
|
table_options_.filter_policy->GetFilterBitsBuilder()); |
||||||
|
builder.Add("foo"); |
||||||
|
builder.Add("bar"); |
||||||
|
builder.Add("box"); |
||||||
|
builder.Add("box"); |
||||||
|
builder.Add("hello"); |
||||||
|
Slice block = builder.Finish(); |
||||||
|
FullFilterBlockReader reader(nullptr, table_options_, block, |
||||||
|
table_options_.filter_policy->GetFilterBitsReader(block)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("bar")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("hello")); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo")); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("missing")); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("other")); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||||
Loading…
Reference in new issue