In particular, we add a new FilterPolicy class. An instance of this class can be supplied in Options when opening a database. If supplied, the instance is used to generate summaries of keys (e.g., a bloom filter) which are placed in sstables. These summaries are consulted by DB::Get() so we can avoid reading sstable blocks that are guaranteed to not contain the key we are looking for. This change provides one implementation of FilterPolicy based on bloom filters. Other changes: - Updated version number to 1.4. - Some build tweaks. - C binding for CompactRange. - A few more benchmarks: deleteseq, deleterandom, readmissing, seekrandom. - Minor .gitignore update.main
Sanjay Ghemawat
13 years ago
parent
bc1ee4d25e
commit
85584d497e
@ -1,5 +1,8 @@ |
||||
build_config.mk |
||||
*.a |
||||
*.o |
||||
*.dylib* |
||||
*.so |
||||
*.so.* |
||||
*_test |
||||
db_bench |
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,70 @@ |
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// A database can be configured with a custom FilterPolicy object.
|
||||
// This object is responsible for creating a small filter from a set
|
||||
// of keys. These filters are stored in leveldb and are consulted
|
||||
// automatically by leveldb to decide whether or not to read some
|
||||
// information from disk. In many cases, a filter can cut down the
|
||||
// number of disk seeks form a handful to a single disk seek per
|
||||
// DB::Get() call.
|
||||
//
|
||||
// Most people will want to use the builtin bloom filter support (see
|
||||
// NewBloomFilterPolicy() below).
|
||||
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ |
||||
#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ |
||||
|
||||
#include <string> |
||||
|
||||
namespace leveldb { |
||||
|
||||
class Slice; |
||||
|
||||
class FilterPolicy { |
||||
public: |
||||
virtual ~FilterPolicy(); |
||||
|
||||
// Return the name of this policy. Note that if the filter encoding
|
||||
// changes in an incompatible way, the name returned by this method
|
||||
// must be changed. Otherwise, old incompatible filters may be
|
||||
// passed to methods of this type.
|
||||
virtual const char* Name() const = 0; |
||||
|
||||
// keys[0,n-1] contains a list of keys (potentially with duplicates)
|
||||
// that are ordered according to the user supplied comparator.
|
||||
// Append a filter that summarizes keys[0,n-1] to *dst.
|
||||
//
|
||||
// Warning: do not change the initial contents of *dst. Instead,
|
||||
// append the newly constructed filter to *dst.
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) |
||||
const = 0; |
||||
|
||||
// "filter" contains the data appended by a preceding call to
|
||||
// CreateFilter() on this class. This method must return true if
|
||||
// the key was in the list of keys passed to CreateFilter().
|
||||
// This method may return true or false if the key was not on the
|
||||
// list, but it should aim to return false with a high probability.
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0; |
||||
}; |
||||
|
||||
// Return a new filter policy that uses a bloom filter with approximately
|
||||
// the specified number of bits per key. A good value for bits_per_key
|
||||
// is 10, which yields a filter with ~ 1% false positive rate.
|
||||
//
|
||||
// Callers must delete the result after any database that is using the
|
||||
// result has been closed.
|
||||
//
|
||||
// Note: if you are using a custom comparator that ignores some parts
|
||||
// of the keys being compared, you must not use NewBloomFilterPolicy()
|
||||
// and must provide your own FilterPolicy that also ignores the
|
||||
// corresponding parts of the keys. For example, if the comparator
|
||||
// ignores trailing spaces, it would be incorrect to use a
|
||||
// FilterPolicy (like NewBloomFilterPolicy) that does not ignore
|
||||
// trailing spaces in keys.
|
||||
extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key); |
||||
|
||||
} |
||||
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
|
||||
@ -0,0 +1,111 @@ |
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "table/filter_block.h" |
||||
|
||||
#include "leveldb/filter_policy.h" |
||||
#include "util/coding.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
// See doc/table_format.txt for an explanation of the filter block format.
|
||||
|
||||
// Generate new filter every 2KB of data
|
||||
static const size_t kFilterBaseLg = 11; |
||||
static const size_t kFilterBase = 1 << kFilterBaseLg; |
||||
|
||||
FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy) |
||||
: policy_(policy) { |
||||
} |
||||
|
||||
void FilterBlockBuilder::StartBlock(uint64_t block_offset) { |
||||
uint64_t filter_index = (block_offset / kFilterBase); |
||||
assert(filter_index >= filter_offsets_.size()); |
||||
while (filter_index > filter_offsets_.size()) { |
||||
GenerateFilter(); |
||||
} |
||||
} |
||||
|
||||
void FilterBlockBuilder::AddKey(const Slice& key) { |
||||
Slice k = key; |
||||
start_.push_back(keys_.size()); |
||||
keys_.append(k.data(), k.size()); |
||||
} |
||||
|
||||
Slice FilterBlockBuilder::Finish() { |
||||
if (!start_.empty()) { |
||||
GenerateFilter(); |
||||
} |
||||
|
||||
// Append array of per-filter offsets
|
||||
const uint32_t array_offset = result_.size(); |
||||
for (size_t i = 0; i < filter_offsets_.size(); i++) { |
||||
PutFixed32(&result_, filter_offsets_[i]); |
||||
} |
||||
|
||||
PutFixed32(&result_, array_offset); |
||||
result_.push_back(kFilterBaseLg); // Save encoding parameter in result
|
||||
return Slice(result_); |
||||
} |
||||
|
||||
void FilterBlockBuilder::GenerateFilter() { |
||||
const size_t num_keys = start_.size(); |
||||
if (num_keys == 0) { |
||||
// Fast path if there are no keys for this filter
|
||||
filter_offsets_.push_back(result_.size()); |
||||
return; |
||||
} |
||||
|
||||
// Make list of keys from flattened key structure
|
||||
start_.push_back(keys_.size()); // Simplify length computation
|
||||
tmp_keys_.resize(num_keys); |
||||
for (size_t i = 0; i < num_keys; i++) { |
||||
const char* base = keys_.data() + start_[i]; |
||||
size_t length = start_[i+1] - start_[i]; |
||||
tmp_keys_[i] = Slice(base, length); |
||||
} |
||||
|
||||
// Generate filter for current set of keys and append to result_.
|
||||
filter_offsets_.push_back(result_.size()); |
||||
policy_->CreateFilter(&tmp_keys_[0], num_keys, &result_); |
||||
|
||||
tmp_keys_.clear(); |
||||
keys_.clear(); |
||||
start_.clear(); |
||||
} |
||||
|
||||
FilterBlockReader::FilterBlockReader(const FilterPolicy* policy, |
||||
const Slice& contents) |
||||
: policy_(policy), |
||||
data_(NULL), |
||||
offset_(NULL), |
||||
num_(0), |
||||
base_lg_(0) { |
||||
size_t n = contents.size(); |
||||
if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array
|
||||
base_lg_ = contents[n-1]; |
||||
uint32_t last_word = DecodeFixed32(contents.data() + n - 5); |
||||
if (last_word > n - 5) return; |
||||
data_ = contents.data(); |
||||
offset_ = data_ + last_word; |
||||
num_ = (n - 5 - last_word) / 4; |
||||
} |
||||
|
||||
bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) { |
||||
uint64_t index = block_offset >> base_lg_; |
||||
if (index < num_) { |
||||
uint32_t start = DecodeFixed32(offset_ + index*4); |
||||
uint32_t limit = DecodeFixed32(offset_ + index*4 + 4); |
||||
if (start <= limit && limit <= (offset_ - data_)) { |
||||
Slice filter = Slice(data_ + start, limit - start); |
||||
return policy_->KeyMayMatch(key, filter); |
||||
} else if (start == limit) { |
||||
// Empty filters do not match any keys
|
||||
return false; |
||||
} |
||||
} |
||||
return true; // Errors are treated as potential matches
|
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,68 @@ |
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// A filter block is stored near the end of a Table file. It contains
|
||||
// filters (e.g., bloom filters) for all data blocks in the table combined
|
||||
// into a single filter block.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ |
||||
#define STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ |
||||
|
||||
#include <stddef.h> |
||||
#include <stdint.h> |
||||
#include <string> |
||||
#include <vector> |
||||
#include "leveldb/slice.h" |
||||
#include "util/hash.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
class FilterPolicy; |
||||
|
||||
// A FilterBlockBuilder is used to construct all of the filters for a
|
||||
// particular Table. It generates a single string which is stored as
|
||||
// a special block in the Table.
|
||||
//
|
||||
// The sequence of calls to FilterBlockBuilder must match the regexp:
|
||||
// (StartBlock AddKey*)* Finish
|
||||
class FilterBlockBuilder { |
||||
public: |
||||
explicit FilterBlockBuilder(const FilterPolicy*); |
||||
|
||||
void StartBlock(uint64_t block_offset); |
||||
void AddKey(const Slice& key); |
||||
Slice Finish(); |
||||
|
||||
private: |
||||
void GenerateFilter(); |
||||
|
||||
const FilterPolicy* policy_; |
||||
std::string keys_; // Flattened key contents
|
||||
std::vector<size_t> start_; // Starting index in keys_ of each key
|
||||
std::string result_; // Filter data computed so far
|
||||
std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument
|
||||
std::vector<uint32_t> filter_offsets_; |
||||
|
||||
// No copying allowed
|
||||
FilterBlockBuilder(const FilterBlockBuilder&); |
||||
void operator=(const FilterBlockBuilder&); |
||||
}; |
||||
|
||||
class FilterBlockReader { |
||||
public: |
||||
// REQUIRES: "contents" and *policy must stay live while *this is live.
|
||||
FilterBlockReader(const FilterPolicy* policy, const Slice& contents); |
||||
bool KeyMayMatch(uint64_t block_offset, const Slice& key); |
||||
|
||||
private: |
||||
const FilterPolicy* policy_; |
||||
const char* data_; // Pointer to filter data (at block-start)
|
||||
const char* offset_; // Pointer to beginning of offset array (at block-end)
|
||||
size_t num_; // Number of entries in offset array
|
||||
size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file)
|
||||
}; |
||||
|
||||
} |
||||
|
||||
#endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_
|
||||
@ -0,0 +1,128 @@ |
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "table/filter_block.h" |
||||
|
||||
#include "leveldb/filter_policy.h" |
||||
#include "util/coding.h" |
||||
#include "util/hash.h" |
||||
#include "util/logging.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
// For testing: emit an array with one hash value per key
|
||||
class TestHashFilter : public FilterPolicy { |
||||
public: |
||||
virtual const char* Name() const { |
||||
return "TestHashFilter"; |
||||
} |
||||
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
||||
for (int i = 0; i < n; i++) { |
||||
uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); |
||||
PutFixed32(dst, h); |
||||
} |
||||
} |
||||
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { |
||||
uint32_t h = Hash(key.data(), key.size(), 1); |
||||
for (int i = 0; i + 4 <= filter.size(); i += 4) { |
||||
if (h == DecodeFixed32(filter.data() + i)) { |
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
}; |
||||
|
||||
class FilterBlockTest { |
||||
public: |
||||
TestHashFilter policy_; |
||||
}; |
||||
|
||||
TEST(FilterBlockTest, EmptyBuilder) { |
||||
FilterBlockBuilder builder(&policy_); |
||||
Slice block = builder.Finish(); |
||||
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); |
||||
FilterBlockReader reader(&policy_, block); |
||||
ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); |
||||
ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); |
||||
} |
||||
|
||||
TEST(FilterBlockTest, SingleChunk) { |
||||
FilterBlockBuilder builder(&policy_); |
||||
builder.StartBlock(100); |
||||
builder.AddKey("foo"); |
||||
builder.AddKey("bar"); |
||||
builder.AddKey("box"); |
||||
builder.StartBlock(200); |
||||
builder.AddKey("box"); |
||||
builder.StartBlock(300); |
||||
builder.AddKey("hello"); |
||||
Slice block = builder.Finish(); |
||||
FilterBlockReader reader(&policy_, block); |
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); |
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "bar")); |
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "box")); |
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "hello")); |
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(100, "missing")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(100, "other")); |
||||
} |
||||
|
||||
TEST(FilterBlockTest, MultiChunk) { |
||||
FilterBlockBuilder builder(&policy_); |
||||
|
||||
// First filter
|
||||
builder.StartBlock(0); |
||||
builder.AddKey("foo"); |
||||
builder.StartBlock(2000); |
||||
builder.AddKey("bar"); |
||||
|
||||
// Second filter
|
||||
builder.StartBlock(3100); |
||||
builder.AddKey("box"); |
||||
|
||||
// Third filter is empty
|
||||
|
||||
// Last filter
|
||||
builder.StartBlock(9000); |
||||
builder.AddKey("box"); |
||||
builder.AddKey("hello"); |
||||
|
||||
Slice block = builder.Finish(); |
||||
FilterBlockReader reader(&policy_, block); |
||||
|
||||
// Check first filter
|
||||
ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); |
||||
ASSERT_TRUE(reader.KeyMayMatch(2000, "bar")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(0, "box")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(0, "hello")); |
||||
|
||||
// Check second filter
|
||||
ASSERT_TRUE(reader.KeyMayMatch(3100, "box")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "foo")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "bar")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "hello")); |
||||
|
||||
// Check third filter (empty)
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "foo")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "bar")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "box")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "hello")); |
||||
|
||||
// Check last filter
|
||||
ASSERT_TRUE(reader.KeyMayMatch(9000, "box")); |
||||
ASSERT_TRUE(reader.KeyMayMatch(9000, "hello")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(9000, "foo")); |
||||
ASSERT_TRUE(! reader.KeyMayMatch(9000, "bar")); |
||||
} |
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return leveldb::test::RunAllTests(); |
||||
} |
||||
@ -0,0 +1,95 @@ |
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "leveldb/filter_policy.h" |
||||
|
||||
#include "leveldb/slice.h" |
||||
#include "util/hash.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
namespace { |
||||
static uint32_t BloomHash(const Slice& key) { |
||||
return Hash(key.data(), key.size(), 0xbc9f1d34); |
||||
} |
||||
|
||||
class BloomFilterPolicy : public FilterPolicy { |
||||
private: |
||||
size_t bits_per_key_; |
||||
size_t k_; |
||||
|
||||
public: |
||||
explicit BloomFilterPolicy(int bits_per_key) |
||||
: bits_per_key_(bits_per_key) { |
||||
// We intentionally round down to reduce probing cost a little bit
|
||||
k_ = static_cast<size_t>(bits_per_key * 0.69); // 0.69 =~ ln(2)
|
||||
if (k_ < 1) k_ = 1; |
||||
if (k_ > 30) k_ = 30; |
||||
} |
||||
|
||||
virtual const char* Name() const { |
||||
return "leveldb.BuiltinBloomFilter"; |
||||
} |
||||
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
||||
// Compute bloom filter size (in both bits and bytes)
|
||||
size_t bits = n * bits_per_key_; |
||||
|
||||
// For small n, we can see a very high false positive rate. Fix it
|
||||
// by enforcing a minimum bloom filter length.
|
||||
if (bits < 64) bits = 64; |
||||
|
||||
size_t bytes = (bits + 7) / 8; |
||||
bits = bytes * 8; |
||||
|
||||
const size_t init_size = dst->size(); |
||||
dst->resize(init_size + bytes, 0); |
||||
dst->push_back(static_cast<char>(k_)); // Remember # of probes in filter
|
||||
char* array = &(*dst)[init_size]; |
||||
for (size_t i = 0; i < n; i++) { |
||||
// Use double-hashing to generate a sequence of hash values.
|
||||
// See analysis in [Kirsch,Mitzenmacher 2006].
|
||||
uint32_t h = BloomHash(keys[i]); |
||||
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
|
||||
for (size_t j = 0; j < k_; j++) { |
||||
const uint32_t bitpos = h % bits; |
||||
array[bitpos/8] |= (1 << (bitpos % 8)); |
||||
h += delta; |
||||
} |
||||
} |
||||
} |
||||
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const { |
||||
const size_t len = bloom_filter.size(); |
||||
if (len < 2) return false; |
||||
|
||||
const char* array = bloom_filter.data(); |
||||
const size_t bits = (len - 1) * 8; |
||||
|
||||
// Use the encoded k so that we can read filters generated by
|
||||
// bloom filters created using different parameters.
|
||||
const size_t k = array[len-1]; |
||||
if (k > 30) { |
||||
// Reserved for potentially new encodings for short bloom filters.
|
||||
// Consider it a match.
|
||||
return true; |
||||
} |
||||
|
||||
uint32_t h = BloomHash(key); |
||||
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
|
||||
for (size_t j = 0; j < k; j++) { |
||||
const uint32_t bitpos = h % bits; |
||||
if ((array[bitpos/8] & (1 << (bitpos % 8))) == 0) return false; |
||||
h += delta; |
||||
} |
||||
return true; |
||||
} |
||||
}; |
||||
} |
||||
|
||||
const FilterPolicy* NewBloomFilterPolicy(int bits_per_key) { |
||||
return new BloomFilterPolicy(bits_per_key); |
||||
} |
||||
|
||||
} // namespace leveldb
|
||||
@ -0,0 +1,159 @@ |
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "leveldb/filter_policy.h" |
||||
|
||||
#include "util/logging.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
static const int kVerbose = 1; |
||||
|
||||
static Slice Key(int i, char* buffer) { |
||||
memcpy(buffer, &i, sizeof(i)); |
||||
return Slice(buffer, sizeof(i)); |
||||
} |
||||
|
||||
class BloomTest { |
||||
private: |
||||
const FilterPolicy* policy_; |
||||
std::string filter_; |
||||
std::vector<std::string> keys_; |
||||
|
||||
public: |
||||
BloomTest() : policy_(NewBloomFilterPolicy(10)) { } |
||||
|
||||
~BloomTest() { |
||||
delete policy_; |
||||
} |
||||
|
||||
void Reset() { |
||||
keys_.clear(); |
||||
filter_.clear(); |
||||
} |
||||
|
||||
void Add(const Slice& s) { |
||||
keys_.push_back(s.ToString()); |
||||
} |
||||
|
||||
void Build() { |
||||
std::vector<Slice> key_slices; |
||||
for (size_t i = 0; i < keys_.size(); i++) { |
||||
key_slices.push_back(Slice(keys_[i])); |
||||
} |
||||
filter_.clear(); |
||||
policy_->CreateFilter(&key_slices[0], key_slices.size(), &filter_); |
||||
keys_.clear(); |
||||
if (kVerbose >= 2) DumpFilter(); |
||||
} |
||||
|
||||
size_t FilterSize() const { |
||||
return filter_.size(); |
||||
} |
||||
|
||||
void DumpFilter() { |
||||
fprintf(stderr, "F("); |
||||
for (size_t i = 0; i+1 < filter_.size(); i++) { |
||||
const unsigned int c = static_cast<unsigned int>(filter_[i]); |
||||
for (int j = 0; j < 8; j++) { |
||||
fprintf(stderr, "%c", (c & (1 <<j)) ? '1' : '.'); |
||||
} |
||||
} |
||||
fprintf(stderr, ")\n"); |
||||
} |
||||
|
||||
bool Matches(const Slice& s) { |
||||
if (!keys_.empty()) { |
||||
Build(); |
||||
} |
||||
return policy_->KeyMayMatch(s, filter_); |
||||
} |
||||
|
||||
double FalsePositiveRate() { |
||||
char buffer[sizeof(int)]; |
||||
int result = 0; |
||||
for (int i = 0; i < 10000; i++) { |
||||
if (Matches(Key(i + 1000000000, buffer))) { |
||||
result++; |
||||
} |
||||
} |
||||
return result / 10000.0; |
||||
} |
||||
}; |
||||
|
||||
TEST(BloomTest, EmptyFilter) { |
||||
ASSERT_TRUE(! Matches("hello")); |
||||
ASSERT_TRUE(! Matches("world")); |
||||
} |
||||
|
||||
TEST(BloomTest, Small) { |
||||
Add("hello"); |
||||
Add("world"); |
||||
ASSERT_TRUE(Matches("hello")); |
||||
ASSERT_TRUE(Matches("world")); |
||||
ASSERT_TRUE(! Matches("x")); |
||||
ASSERT_TRUE(! Matches("foo")); |
||||
} |
||||
|
||||
static int NextLength(int length) { |
||||
if (length < 10) { |
||||
length += 1; |
||||
} else if (length < 100) { |
||||
length += 10; |
||||
} else if (length < 1000) { |
||||
length += 100; |
||||
} else { |
||||
length += 1000; |
||||
} |
||||
return length; |
||||
} |
||||
|
||||
TEST(BloomTest, VaryingLengths) { |
||||
char buffer[sizeof(int)]; |
||||
|
||||
// Count number of filters that significantly exceed the false positive rate
|
||||
int mediocre_filters = 0; |
||||
int good_filters = 0; |
||||
|
||||
for (int length = 1; length <= 10000; length = NextLength(length)) { |
||||
Reset(); |
||||
for (int i = 0; i < length; i++) { |
||||
Add(Key(i, buffer)); |
||||
} |
||||
Build(); |
||||
|
||||
ASSERT_LE(FilterSize(), (length * 10 / 8) + 40) << length; |
||||
|
||||
// All added keys must match
|
||||
for (int i = 0; i < length; i++) { |
||||
ASSERT_TRUE(Matches(Key(i, buffer))) |
||||
<< "Length " << length << "; key " << i; |
||||
} |
||||
|
||||
// Check false positive rate
|
||||
double rate = FalsePositiveRate(); |
||||
if (kVerbose >= 1) { |
||||
fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n", |
||||
rate*100.0, length, static_cast<int>(FilterSize())); |
||||
} |
||||
ASSERT_LE(rate, 0.02); // Must not be over 2%
|
||||
if (rate > 0.0125) mediocre_filters++; // Allowed, but not too often
|
||||
else good_filters++; |
||||
} |
||||
if (kVerbose >= 1) { |
||||
fprintf(stderr, "Filters: %d good, %d mediocre\n", |
||||
good_filters, mediocre_filters); |
||||
} |
||||
ASSERT_LE(mediocre_filters, good_filters/5); |
||||
} |
||||
|
||||
// Different bits-per-byte
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return leveldb::test::RunAllTests(); |
||||
} |
||||
@ -0,0 +1,11 @@ |
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "leveldb/filter_policy.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
FilterPolicy::~FilterPolicy() { } |
||||
|
||||
} // namespace leveldb
|
||||
Loading…
Reference in new issue