Conflicts: HISTORY.md db/db_impl.cc db/db_impl.h db/db_iter.cc db/db_test.cc db/dbformat.h db/memtable.cc db/memtable_list.cc db/memtable_list.h db/table_cache.cc db/table_cache.h db/version_edit.h db/version_set.cc db/version_set.h db/write_batch.cc db/write_batch_test.cc include/rocksdb/options.h util/options.ccmain
Igor Canadi
11 years ago
commit
0143abdbb0
@ -0,0 +1,337 @@ |
|||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
#include <algorithm> |
||||||
|
#include <set> |
||||||
|
|
||||||
|
#include "db/db_impl.h" |
||||||
|
#include "db/filename.h" |
||||||
|
#include "db/version_set.h" |
||||||
|
#include "db/write_batch_internal.h" |
||||||
|
#include "rocksdb/cache.h" |
||||||
|
#include "rocksdb/compaction_filter.h" |
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/filter_policy.h" |
||||||
|
#include "rocksdb/slice_transform.h" |
||||||
|
#include "rocksdb/table.h" |
||||||
|
#include "table/plain_table_factory.h" |
||||||
|
#include "util/hash.h" |
||||||
|
#include "util/logging.h" |
||||||
|
#include "util/mutexlock.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
#include "utilities/merge_operators.h" |
||||||
|
|
||||||
|
using std::unique_ptr; |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class PlainTableDBTest { |
||||||
|
protected: |
||||||
|
private: |
||||||
|
std::string dbname_; |
||||||
|
Env* env_; |
||||||
|
DB* db_; |
||||||
|
|
||||||
|
Options last_options_; |
||||||
|
static std::unique_ptr<const SliceTransform> prefix_transform; |
||||||
|
|
||||||
|
public: |
||||||
|
PlainTableDBTest() : env_(Env::Default()) { |
||||||
|
dbname_ = test::TmpDir() + "/plain_table_db_test"; |
||||||
|
ASSERT_OK(DestroyDB(dbname_, Options())); |
||||||
|
db_ = nullptr; |
||||||
|
Reopen(); |
||||||
|
} |
||||||
|
|
||||||
|
~PlainTableDBTest() { |
||||||
|
delete db_; |
||||||
|
ASSERT_OK(DestroyDB(dbname_, Options())); |
||||||
|
} |
||||||
|
|
||||||
|
// Return the current option configuration.
|
||||||
|
Options CurrentOptions() { |
||||||
|
Options options; |
||||||
|
options.table_factory.reset(new PlainTableFactory(16, 2, 0.8)); |
||||||
|
options.prefix_extractor = prefix_transform.get(); |
||||||
|
options.allow_mmap_reads = true; |
||||||
|
return options; |
||||||
|
} |
||||||
|
|
||||||
|
DBImpl* dbfull() { |
||||||
|
return reinterpret_cast<DBImpl*>(db_); |
||||||
|
} |
||||||
|
|
||||||
|
void Reopen(Options* options = nullptr) { |
||||||
|
ASSERT_OK(TryReopen(options)); |
||||||
|
} |
||||||
|
|
||||||
|
void Close() { |
||||||
|
delete db_; |
||||||
|
db_ = nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
void DestroyAndReopen(Options* options = nullptr) { |
||||||
|
//Destroy using last options
|
||||||
|
Destroy(&last_options_); |
||||||
|
ASSERT_OK(TryReopen(options)); |
||||||
|
} |
||||||
|
|
||||||
|
void Destroy(Options* options) { |
||||||
|
delete db_; |
||||||
|
db_ = nullptr; |
||||||
|
ASSERT_OK(DestroyDB(dbname_, *options)); |
||||||
|
} |
||||||
|
|
||||||
|
Status PureReopen(Options* options, DB** db) { |
||||||
|
return DB::Open(*options, dbname_, db); |
||||||
|
} |
||||||
|
|
||||||
|
Status TryReopen(Options* options = nullptr) { |
||||||
|
delete db_; |
||||||
|
db_ = nullptr; |
||||||
|
Options opts; |
||||||
|
if (options != nullptr) { |
||||||
|
opts = *options; |
||||||
|
} else { |
||||||
|
opts = CurrentOptions(); |
||||||
|
opts.create_if_missing = true; |
||||||
|
} |
||||||
|
last_options_ = opts; |
||||||
|
|
||||||
|
return DB::Open(opts, dbname_, &db_); |
||||||
|
} |
||||||
|
|
||||||
|
Status Put(const Slice& k, const Slice& v) { |
||||||
|
return db_->Put(WriteOptions(), k, v); |
||||||
|
} |
||||||
|
|
||||||
|
Status Delete(const std::string& k) { |
||||||
|
return db_->Delete(WriteOptions(), k); |
||||||
|
} |
||||||
|
|
||||||
|
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { |
||||||
|
ReadOptions options; |
||||||
|
options.snapshot = snapshot; |
||||||
|
std::string result; |
||||||
|
Status s = db_->Get(options, k, &result); |
||||||
|
if (s.IsNotFound()) { |
||||||
|
result = "NOT_FOUND"; |
||||||
|
} else if (!s.ok()) { |
||||||
|
result = s.ToString(); |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
int NumTableFilesAtLevel(int level) { |
||||||
|
std::string property; |
||||||
|
ASSERT_TRUE( |
||||||
|
db_->GetProperty("rocksdb.num-files-at-level" + NumberToString(level), |
||||||
|
&property)); |
||||||
|
return atoi(property.c_str()); |
||||||
|
} |
||||||
|
|
||||||
|
// Return spread of files per level
|
||||||
|
std::string FilesPerLevel() { |
||||||
|
std::string result; |
||||||
|
int last_non_zero_offset = 0; |
||||||
|
for (int level = 0; level < db_->NumberLevels(); level++) { |
||||||
|
int f = NumTableFilesAtLevel(level); |
||||||
|
char buf[100]; |
||||||
|
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); |
||||||
|
result += buf; |
||||||
|
if (f > 0) { |
||||||
|
last_non_zero_offset = result.size(); |
||||||
|
} |
||||||
|
} |
||||||
|
result.resize(last_non_zero_offset); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
std::string IterStatus(Iterator* iter) { |
||||||
|
std::string result; |
||||||
|
if (iter->Valid()) { |
||||||
|
result = iter->key().ToString() + "->" + iter->value().ToString(); |
||||||
|
} else { |
||||||
|
result = "(invalid)"; |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
std::unique_ptr<const SliceTransform> PlainTableDBTest::prefix_transform( |
||||||
|
NewFixedPrefixTransform(8)); |
||||||
|
|
||||||
|
TEST(PlainTableDBTest, Empty) { |
||||||
|
ASSERT_TRUE(dbfull() != nullptr); |
||||||
|
ASSERT_EQ("NOT_FOUND", Get("0000000000000foo")); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(PlainTableDBTest, ReadWrite) { |
||||||
|
ASSERT_OK(Put("1000000000000foo", "v1")); |
||||||
|
ASSERT_EQ("v1", Get("1000000000000foo")); |
||||||
|
ASSERT_OK(Put("0000000000000bar", "v2")); |
||||||
|
ASSERT_OK(Put("1000000000000foo", "v3")); |
||||||
|
ASSERT_EQ("v3", Get("1000000000000foo")); |
||||||
|
ASSERT_EQ("v2", Get("0000000000000bar")); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(PlainTableDBTest, Flush) { |
||||||
|
ASSERT_OK(Put("1000000000000foo", "v1")); |
||||||
|
ASSERT_OK(Put("0000000000000bar", "v2")); |
||||||
|
ASSERT_OK(Put("1000000000000foo", "v3")); |
||||||
|
dbfull()->TEST_FlushMemTable(); |
||||||
|
ASSERT_EQ("v3", Get("1000000000000foo")); |
||||||
|
ASSERT_EQ("v2", Get("0000000000000bar")); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(PlainTableDBTest, Iterator) { |
||||||
|
ASSERT_OK(Put("1000000000foo002", "v_2")); |
||||||
|
ASSERT_OK(Put("0000000000000bar", "random")); |
||||||
|
ASSERT_OK(Put("1000000000foo001", "v1")); |
||||||
|
ASSERT_OK(Put("3000000000000bar", "bar_v")); |
||||||
|
ASSERT_OK(Put("1000000000foo003", "v__3")); |
||||||
|
ASSERT_OK(Put("1000000000foo004", "v__4")); |
||||||
|
ASSERT_OK(Put("1000000000foo005", "v__5")); |
||||||
|
ASSERT_OK(Put("1000000000foo007", "v__7")); |
||||||
|
ASSERT_OK(Put("1000000000foo008", "v__8")); |
||||||
|
dbfull()->TEST_FlushMemTable(); |
||||||
|
ASSERT_EQ("v1", Get("1000000000foo001")); |
||||||
|
ASSERT_EQ("v__3", Get("1000000000foo003")); |
||||||
|
ReadOptions ro; |
||||||
|
Iterator* iter = dbfull()->NewIterator(ro); |
||||||
|
iter->Seek("1000000000foo001"); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo001", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v1", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Next(); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo002", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v_2", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Next(); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo003", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v__3", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Next(); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo004", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v__4", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Seek("3000000000000bar"); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("3000000000000bar", iter->key().ToString()); |
||||||
|
ASSERT_EQ("bar_v", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Seek("1000000000foo000"); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo001", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v1", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Seek("1000000000foo005"); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo005", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v__5", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Seek("1000000000foo006"); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo007", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v__7", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Seek("1000000000foo008"); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("1000000000foo008", iter->key().ToString()); |
||||||
|
ASSERT_EQ("v__8", iter->value().ToString()); |
||||||
|
|
||||||
|
iter->Seek("1000000000foo009"); |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
ASSERT_EQ("3000000000000bar", iter->key().ToString()); |
||||||
|
|
||||||
|
|
||||||
|
delete iter; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(PlainTableDBTest, Flush2) { |
||||||
|
ASSERT_OK(Put("0000000000000bar", "b")); |
||||||
|
ASSERT_OK(Put("1000000000000foo", "v1")); |
||||||
|
dbfull()->TEST_FlushMemTable(); |
||||||
|
|
||||||
|
ASSERT_OK(Put("1000000000000foo", "v2")); |
||||||
|
dbfull()->TEST_FlushMemTable(); |
||||||
|
ASSERT_EQ("v2", Get("1000000000000foo")); |
||||||
|
|
||||||
|
ASSERT_OK(Put("0000000000000eee", "v3")); |
||||||
|
dbfull()->TEST_FlushMemTable(); |
||||||
|
ASSERT_EQ("v3", Get("0000000000000eee")); |
||||||
|
|
||||||
|
ASSERT_OK(Delete("0000000000000bar")); |
||||||
|
dbfull()->TEST_FlushMemTable(); |
||||||
|
ASSERT_EQ("NOT_FOUND", Get("0000000000000bar")); |
||||||
|
|
||||||
|
ASSERT_OK(Put("0000000000000eee", "v5")); |
||||||
|
dbfull()->TEST_FlushMemTable(); |
||||||
|
ASSERT_EQ("v5", Get("0000000000000eee")); |
||||||
|
} |
||||||
|
|
||||||
|
static std::string Key(int i) { |
||||||
|
char buf[100]; |
||||||
|
snprintf(buf, sizeof(buf), "key_______%06d", i); |
||||||
|
return std::string(buf); |
||||||
|
} |
||||||
|
|
||||||
|
static std::string RandomString(Random* rnd, int len) { |
||||||
|
std::string r; |
||||||
|
test::RandomString(rnd, len, &r); |
||||||
|
return r; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(PlainTableDBTest, CompactionTrigger) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.write_buffer_size = 100 << 10; //100KB
|
||||||
|
options.num_levels = 3; |
||||||
|
options.max_mem_compaction_level = 0; |
||||||
|
options.level0_file_num_compaction_trigger = 3; |
||||||
|
Reopen(&options); |
||||||
|
|
||||||
|
Random rnd(301); |
||||||
|
|
||||||
|
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; |
||||||
|
num++) { |
||||||
|
std::vector<std::string> values; |
||||||
|
// Write 120KB (12 values, each 10K)
|
||||||
|
for (int i = 0; i < 12; i++) { |
||||||
|
values.push_back(RandomString(&rnd, 10000)); |
||||||
|
ASSERT_OK(Put(Key(i), values[i])); |
||||||
|
} |
||||||
|
dbfull()->TEST_WaitForFlushMemTable(); |
||||||
|
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1); |
||||||
|
} |
||||||
|
|
||||||
|
//generate one more file in level-0, and should trigger level-0 compaction
|
||||||
|
std::vector<std::string> values; |
||||||
|
for (int i = 0; i < 12; i++) { |
||||||
|
values.push_back(RandomString(&rnd, 10000)); |
||||||
|
ASSERT_OK(Put(Key(i), values[i])); |
||||||
|
} |
||||||
|
dbfull()->TEST_WaitForCompact(); |
||||||
|
|
||||||
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0); |
||||||
|
ASSERT_EQ(NumTableFilesAtLevel(1), 1); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
return rocksdb::test::RunAllTests(); |
||||||
|
} |
||||||
@ -1,45 +0,0 @@ |
|||||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under the BSD-style license found in the
|
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
|
||||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a BSD-style license that can be
|
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
||||||
//
|
|
||||||
// Arena class defines memory allocation methods. It's used by memtable and
|
|
||||||
// skiplist.
|
|
||||||
|
|
||||||
#ifndef STORAGE_ROCKSDB_INCLUDE_ARENA_H_ |
|
||||||
#define STORAGE_ROCKSDB_INCLUDE_ARENA_H_ |
|
||||||
|
|
||||||
#include <limits> |
|
||||||
#include <memory> |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
class Arena { |
|
||||||
public: |
|
||||||
Arena() {}; |
|
||||||
virtual ~Arena() {}; |
|
||||||
|
|
||||||
// Return a pointer to a newly allocated memory block of "bytes" bytes.
|
|
||||||
virtual char* Allocate(size_t bytes) = 0; |
|
||||||
|
|
||||||
// Allocate memory with the normal alignment guarantees provided by malloc.
|
|
||||||
virtual char* AllocateAligned(size_t bytes) = 0; |
|
||||||
|
|
||||||
// Returns an estimate of the total memory used by arena.
|
|
||||||
virtual const size_t ApproximateMemoryUsage() = 0; |
|
||||||
|
|
||||||
// Returns the total number of bytes in all blocks allocated so far.
|
|
||||||
virtual const size_t MemoryAllocatedBytes() = 0; |
|
||||||
|
|
||||||
private: |
|
||||||
// No copying allowed
|
|
||||||
Arena(const Arena&); |
|
||||||
void operator=(const Arena&); |
|
||||||
}; |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
|
|
||||||
#endif // STORAGE_ROCKSDB_INCLUDE_ARENA_H_
|
|
||||||
@ -1,31 +0,0 @@ |
|||||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under the BSD-style license found in the
|
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
|
||||||
|
|
||||||
#pragma once |
|
||||||
#include <memory> |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
class FlushBlockPolicyFactory; |
|
||||||
|
|
||||||
struct BlockBasedTableOptions { |
|
||||||
// @flush_block_policy_factory creates the instances of flush block policy.
|
|
||||||
// which provides a configurable way to determine when to flush a block in
|
|
||||||
// the block based tables. If not set, table builder will use the default
|
|
||||||
// block flush policy, which cut blocks by block size (please refer to
|
|
||||||
// `FlushBlockBySizePolicy`).
|
|
||||||
std::shared_ptr<FlushBlockPolicyFactory> flush_block_policy_factory; |
|
||||||
|
|
||||||
// TODO(kailiu) Temporarily disable this feature by making the default value
|
|
||||||
// to be false. Also in master branch, this file is non-public so no user
|
|
||||||
// will be able to change the value of `cache_index_and_filter_blocks`.
|
|
||||||
//
|
|
||||||
// Indicating if we'd put index/filter blocks to the block cache.
|
|
||||||
// If not specified, each "table reader" object will pre-load index/filter
|
|
||||||
// block during table initialization.
|
|
||||||
bool cache_index_and_filter_blocks = false; |
|
||||||
}; |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
@ -0,0 +1,286 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "table/meta_blocks.h" |
||||||
|
|
||||||
|
#include <map> |
||||||
|
|
||||||
|
#include "rocksdb/table.h" |
||||||
|
#include "table/block.h" |
||||||
|
#include "table/format.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
MetaIndexBuilder::MetaIndexBuilder() |
||||||
|
: meta_index_block_( |
||||||
|
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { |
||||||
|
} |
||||||
|
|
||||||
|
void MetaIndexBuilder::Add(const std::string& key, |
||||||
|
const BlockHandle& handle) { |
||||||
|
std::string handle_encoding; |
||||||
|
handle.EncodeTo(&handle_encoding); |
||||||
|
meta_block_handles_.insert({key, handle_encoding}); |
||||||
|
} |
||||||
|
|
||||||
|
Slice MetaIndexBuilder::Finish() { |
||||||
|
for (const auto& metablock : meta_block_handles_) { |
||||||
|
meta_index_block_->Add(metablock.first, metablock.second); |
||||||
|
} |
||||||
|
return meta_index_block_->Finish(); |
||||||
|
} |
||||||
|
|
||||||
|
PropertyBlockBuilder::PropertyBlockBuilder() |
||||||
|
: properties_block_( |
||||||
|
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::Add(const std::string& name, |
||||||
|
const std::string& val) { |
||||||
|
props_.insert({name, val}); |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::Add(const std::string& name, uint64_t val) { |
||||||
|
assert(props_.find(name) == props_.end()); |
||||||
|
|
||||||
|
std::string dst; |
||||||
|
PutVarint64(&dst, val); |
||||||
|
|
||||||
|
Add(name, dst); |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::Add( |
||||||
|
const UserCollectedProperties& user_collected_properties) { |
||||||
|
for (const auto& prop : user_collected_properties) { |
||||||
|
Add(prop.first, prop.second); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { |
||||||
|
Add(TablePropertiesNames::kRawKeySize, props.raw_key_size); |
||||||
|
Add(TablePropertiesNames::kRawValueSize, props.raw_value_size); |
||||||
|
Add(TablePropertiesNames::kDataSize, props.data_size); |
||||||
|
Add(TablePropertiesNames::kIndexSize, props.index_size); |
||||||
|
Add(TablePropertiesNames::kNumEntries, props.num_entries); |
||||||
|
Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks); |
||||||
|
Add(TablePropertiesNames::kFilterSize, props.filter_size); |
||||||
|
Add(TablePropertiesNames::kFormatVersion, props.format_version); |
||||||
|
Add(TablePropertiesNames::kFixedKeyLen, props.fixed_key_len); |
||||||
|
|
||||||
|
if (!props.filter_policy_name.empty()) { |
||||||
|
Add(TablePropertiesNames::kFilterPolicy, |
||||||
|
props.filter_policy_name); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Slice PropertyBlockBuilder::Finish() { |
||||||
|
for (const auto& prop : props_) { |
||||||
|
properties_block_->Add(prop.first, prop.second); |
||||||
|
} |
||||||
|
|
||||||
|
return properties_block_->Finish(); |
||||||
|
} |
||||||
|
|
||||||
|
void LogPropertiesCollectionError( |
||||||
|
Logger* info_log, const std::string& method, const std::string& name) { |
||||||
|
assert(method == "Add" || method == "Finish"); |
||||||
|
|
||||||
|
std::string msg = |
||||||
|
"[Warning] encountered error when calling TablePropertiesCollector::" + |
||||||
|
method + "() with collector name: " + name; |
||||||
|
Log(info_log, "%s", msg.c_str()); |
||||||
|
} |
||||||
|
|
||||||
|
bool NotifyCollectTableCollectorsOnAdd( |
||||||
|
const Slice& key, |
||||||
|
const Slice& value, |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log) { |
||||||
|
bool all_succeeded = true; |
||||||
|
for (auto collector : collectors) { |
||||||
|
Status s = collector->Add(key, value); |
||||||
|
all_succeeded = all_succeeded && s.ok(); |
||||||
|
if (!s.ok()) { |
||||||
|
LogPropertiesCollectionError( |
||||||
|
info_log, "Add", /* method */ collector->Name() |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
return all_succeeded; |
||||||
|
} |
||||||
|
|
||||||
|
bool NotifyCollectTableCollectorsOnFinish( |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log, |
||||||
|
PropertyBlockBuilder* builder) { |
||||||
|
bool all_succeeded = true; |
||||||
|
for (auto collector : collectors) { |
||||||
|
UserCollectedProperties user_collected_properties; |
||||||
|
Status s = collector->Finish(&user_collected_properties); |
||||||
|
|
||||||
|
all_succeeded = all_succeeded && s.ok(); |
||||||
|
if (!s.ok()) { |
||||||
|
LogPropertiesCollectionError( |
||||||
|
info_log, "Finish", /* method */ collector->Name() |
||||||
|
); |
||||||
|
} else { |
||||||
|
builder->Add(user_collected_properties); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return all_succeeded; |
||||||
|
} |
||||||
|
|
||||||
|
Status ReadProperties( |
||||||
|
const Slice& handle_value, |
||||||
|
RandomAccessFile* file, |
||||||
|
Env* env, |
||||||
|
Logger* logger, |
||||||
|
TableProperties* table_properties) { |
||||||
|
assert(table_properties); |
||||||
|
|
||||||
|
Slice v = handle_value; |
||||||
|
BlockHandle handle; |
||||||
|
if (!handle.DecodeFrom(&v).ok()) { |
||||||
|
return Status::InvalidArgument("Failed to decode properties block handle"); |
||||||
|
} |
||||||
|
|
||||||
|
BlockContents block_contents; |
||||||
|
ReadOptions read_options; |
||||||
|
read_options.verify_checksums = false; |
||||||
|
Status s = ReadBlockContents( |
||||||
|
file, |
||||||
|
read_options, |
||||||
|
handle, |
||||||
|
&block_contents, |
||||||
|
env, |
||||||
|
false |
||||||
|
); |
||||||
|
|
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Block properties_block(block_contents); |
||||||
|
std::unique_ptr<Iterator> iter( |
||||||
|
properties_block.NewIterator(BytewiseComparator()) |
||||||
|
); |
||||||
|
|
||||||
|
// All pre-defined properties of type uint64_t
|
||||||
|
std::unordered_map<std::string, uint64_t*> predefined_uint64_properties = { |
||||||
|
{ TablePropertiesNames::kDataSize, &table_properties->data_size }, |
||||||
|
{ TablePropertiesNames::kIndexSize, &table_properties->index_size }, |
||||||
|
{ TablePropertiesNames::kFilterSize, &table_properties->filter_size }, |
||||||
|
{ TablePropertiesNames::kRawKeySize, &table_properties->raw_key_size }, |
||||||
|
{ TablePropertiesNames::kRawValueSize, &table_properties->raw_value_size }, |
||||||
|
{ TablePropertiesNames::kNumDataBlocks, |
||||||
|
&table_properties->num_data_blocks }, |
||||||
|
{ TablePropertiesNames::kNumEntries, &table_properties->num_entries }, |
||||||
|
{ TablePropertiesNames::kFormatVersion, &table_properties->format_version }, |
||||||
|
{ TablePropertiesNames::kFixedKeyLen, &table_properties->fixed_key_len }, |
||||||
|
}; |
||||||
|
|
||||||
|
std::string last_key; |
||||||
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { |
||||||
|
s = iter->status(); |
||||||
|
if (!s.ok()) { |
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
auto key = iter->key().ToString(); |
||||||
|
// properties block is strictly sorted with no duplicate key.
|
||||||
|
assert( |
||||||
|
last_key.empty() || |
||||||
|
BytewiseComparator()->Compare(key, last_key) > 0 |
||||||
|
); |
||||||
|
last_key = key; |
||||||
|
|
||||||
|
auto raw_val = iter->value(); |
||||||
|
auto pos = predefined_uint64_properties.find(key); |
||||||
|
|
||||||
|
if (pos != predefined_uint64_properties.end()) { |
||||||
|
// handle predefined rocksdb properties
|
||||||
|
uint64_t val; |
||||||
|
if (!GetVarint64(&raw_val, &val)) { |
||||||
|
// skip malformed value
|
||||||
|
auto error_msg = |
||||||
|
"[Warning] detect malformed value in properties meta-block:" |
||||||
|
"\tkey: " + key + "\tval: " + raw_val.ToString(); |
||||||
|
Log(logger, "%s", error_msg.c_str()); |
||||||
|
continue; |
||||||
|
} |
||||||
|
*(pos->second) = val; |
||||||
|
} else if (key == TablePropertiesNames::kFilterPolicy) { |
||||||
|
table_properties->filter_policy_name = raw_val.ToString(); |
||||||
|
} else { |
||||||
|
// handle user-collected properties
|
||||||
|
table_properties->user_collected_properties.insert( |
||||||
|
std::make_pair(key, raw_val.ToString()) |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Status ReadTableProperties( |
||||||
|
RandomAccessFile* file, |
||||||
|
uint64_t file_size, |
||||||
|
uint64_t table_magic_number, |
||||||
|
Env* env, |
||||||
|
Logger* info_log, |
||||||
|
TableProperties* properties) { |
||||||
|
// -- Read metaindex block
|
||||||
|
Footer footer(table_magic_number); |
||||||
|
auto s = ReadFooterFromFile(file, file_size, &footer); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
auto metaindex_handle = footer.metaindex_handle(); |
||||||
|
BlockContents metaindex_contents; |
||||||
|
ReadOptions read_options; |
||||||
|
read_options.verify_checksums = false; |
||||||
|
s = ReadBlockContents( |
||||||
|
file, |
||||||
|
read_options, |
||||||
|
metaindex_handle, |
||||||
|
&metaindex_contents, |
||||||
|
env, |
||||||
|
false |
||||||
|
); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
Block metaindex_block(metaindex_contents); |
||||||
|
std::unique_ptr<Iterator> meta_iter( |
||||||
|
metaindex_block.NewIterator(BytewiseComparator()) |
||||||
|
); |
||||||
|
|
||||||
|
// -- Read property block
|
||||||
|
meta_iter->Seek(kPropertiesBlock); |
||||||
|
TableProperties table_properties; |
||||||
|
if (meta_iter->Valid() && |
||||||
|
meta_iter->key() == kPropertiesBlock && |
||||||
|
meta_iter->status().ok()) { |
||||||
|
s = ReadProperties( |
||||||
|
meta_iter->value(), |
||||||
|
file, |
||||||
|
env, |
||||||
|
info_log, |
||||||
|
properties |
||||||
|
); |
||||||
|
} else { |
||||||
|
s = Status::Corruption( |
||||||
|
"Unable to read the property block from the plain table" |
||||||
|
); |
||||||
|
} |
||||||
|
|
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,121 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <map> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "rocksdb/comparator.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/table_properties.h" |
||||||
|
#include "table/block_builder.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class BlockBuilder; |
||||||
|
class BlockHandle; |
||||||
|
class Env; |
||||||
|
class Logger; |
||||||
|
class RandomAccessFile; |
||||||
|
struct TableProperties; |
||||||
|
|
||||||
|
// An STL style comparator that does the bytewise comparator comparasion
|
||||||
|
// internally.
|
||||||
|
struct BytewiseLessThan { |
||||||
|
bool operator()(const std::string& key1, const std::string& key2) const { |
||||||
|
// smaller entries will be placed in front.
|
||||||
|
return comparator->Compare(key1, key2) <= 0; |
||||||
|
} |
||||||
|
|
||||||
|
const Comparator* comparator = BytewiseComparator(); |
||||||
|
}; |
||||||
|
|
||||||
|
// When writing to a block that requires entries to be sorted by
|
||||||
|
// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap`
|
||||||
|
// before writng to store.
|
||||||
|
typedef std::map<std::string, std::string, BytewiseLessThan> BytewiseSortedMap; |
||||||
|
|
||||||
|
class MetaIndexBuilder { |
||||||
|
public: |
||||||
|
MetaIndexBuilder(const MetaIndexBuilder&) = delete; |
||||||
|
MetaIndexBuilder& operator=(const MetaIndexBuilder&) = delete; |
||||||
|
|
||||||
|
MetaIndexBuilder(); |
||||||
|
void Add(const std::string& key, const BlockHandle& handle); |
||||||
|
|
||||||
|
// Write all the added key/value pairs to the block and return the contents
|
||||||
|
// of the block.
|
||||||
|
Slice Finish(); |
||||||
|
|
||||||
|
private: |
||||||
|
// store the sorted key/handle of the metablocks.
|
||||||
|
BytewiseSortedMap meta_block_handles_; |
||||||
|
std::unique_ptr<BlockBuilder> meta_index_block_; |
||||||
|
}; |
||||||
|
|
||||||
|
class PropertyBlockBuilder { |
||||||
|
public: |
||||||
|
PropertyBlockBuilder(const PropertyBlockBuilder&) = delete; |
||||||
|
PropertyBlockBuilder& operator=(const PropertyBlockBuilder&) = delete; |
||||||
|
|
||||||
|
PropertyBlockBuilder(); |
||||||
|
|
||||||
|
void AddTableProperty(const TableProperties& props); |
||||||
|
void Add(const std::string& key, uint64_t value); |
||||||
|
void Add(const std::string& key, const std::string& value); |
||||||
|
void Add(const UserCollectedProperties& user_collected_properties); |
||||||
|
|
||||||
|
// Write all the added entries to the block and return the block contents
|
||||||
|
Slice Finish(); |
||||||
|
|
||||||
|
private: |
||||||
|
std::unique_ptr<BlockBuilder> properties_block_; |
||||||
|
BytewiseSortedMap props_; |
||||||
|
}; |
||||||
|
|
||||||
|
// Were we encounter any error occurs during user-defined statistics collection,
|
||||||
|
// we'll write the warning message to info log.
|
||||||
|
void LogPropertiesCollectionError( |
||||||
|
Logger* info_log, const std::string& method, const std::string& name); |
||||||
|
|
||||||
|
// Utility functions help table builder to trigger batch events for user
|
||||||
|
// defined property collectors.
|
||||||
|
// Return value indicates if there is any error occurred; if error occurred,
|
||||||
|
// the warning message will be logged.
|
||||||
|
// NotifyCollectTableCollectorsOnAdd() triggers the `Add` event for all
|
||||||
|
// property collectors.
|
||||||
|
bool NotifyCollectTableCollectorsOnAdd( |
||||||
|
const Slice& key, |
||||||
|
const Slice& value, |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log); |
||||||
|
|
||||||
|
// NotifyCollectTableCollectorsOnAdd() triggers the `Finish` event for all
|
||||||
|
// property collectors. The collected properties will be added to `builder`.
|
||||||
|
bool NotifyCollectTableCollectorsOnFinish( |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log, |
||||||
|
PropertyBlockBuilder* builder); |
||||||
|
|
||||||
|
// Read the properties from the table.
|
||||||
|
Status ReadProperties( |
||||||
|
const Slice& handle_value, |
||||||
|
RandomAccessFile* file, |
||||||
|
Env* env, |
||||||
|
Logger* logger, |
||||||
|
TableProperties* table_properties); |
||||||
|
|
||||||
|
// Directly read the properties from the properties block of a plain table.
|
||||||
|
Status ReadTableProperties( |
||||||
|
RandomAccessFile* file, |
||||||
|
uint64_t file_size, |
||||||
|
uint64_t table_magic_number, |
||||||
|
Env* env, |
||||||
|
Logger* info_log, |
||||||
|
TableProperties* properties); |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,198 @@ |
|||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "table/plain_table_builder.h" |
||||||
|
|
||||||
|
#include <assert.h> |
||||||
|
#include <map> |
||||||
|
|
||||||
|
#include "rocksdb/comparator.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/filter_policy.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "table/plain_table_factory.h" |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "table/block_builder.h" |
||||||
|
#include "table/filter_block.h" |
||||||
|
#include "table/format.h" |
||||||
|
#include "table/meta_blocks.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/crc32c.h" |
||||||
|
#include "util/stop_watch.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
// a utility that helps writing block content to the file
|
||||||
|
// @offset will advance if @block_contents was successfully written.
|
||||||
|
// @block_handle the block handle this particular block.
|
||||||
|
Status WriteBlock( |
||||||
|
const Slice& block_contents, |
||||||
|
WritableFile* file, |
||||||
|
uint64_t* offset, |
||||||
|
BlockHandle* block_handle) { |
||||||
|
block_handle->set_offset(*offset); |
||||||
|
block_handle->set_size(block_contents.size()); |
||||||
|
Status s = file->Append(block_contents); |
||||||
|
|
||||||
|
if (s.ok()) { |
||||||
|
*offset += block_contents.size(); |
||||||
|
} |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
// kPlainTableMagicNumber was picked by running
|
||||||
|
// echo rocksdb.plain.table | sha1sum
|
||||||
|
// and taking the leading 64 bits.
|
||||||
|
extern const uint64_t kPlainTableMagicNumber = 0x4f3418eb7a8f13b8ull; |
||||||
|
|
||||||
|
PlainTableBuilder::PlainTableBuilder(const Options& options, |
||||||
|
WritableFile* file, |
||||||
|
uint32_t user_key_len) : |
||||||
|
options_(options), file_(file), user_key_len_(user_key_len) { |
||||||
|
properties_.fixed_key_len = user_key_len; |
||||||
|
|
||||||
|
// for plain table, we put all the data in a big chuck.
|
||||||
|
properties_.num_data_blocks = 1; |
||||||
|
// emphasize that currently plain table doesn't have persistent index or
|
||||||
|
// filter block.
|
||||||
|
properties_.index_size = 0; |
||||||
|
properties_.filter_size = 0; |
||||||
|
properties_.format_version = 0; |
||||||
|
} |
||||||
|
|
||||||
|
PlainTableBuilder::~PlainTableBuilder() { |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableBuilder::Add(const Slice& key, const Slice& value) { |
||||||
|
size_t user_key_size = key.size() - 8; |
||||||
|
assert(user_key_len_ == 0 || user_key_size == user_key_len_); |
||||||
|
|
||||||
|
if (!IsFixedLength()) { |
||||||
|
// Write key length
|
||||||
|
key_size_str_.clear(); |
||||||
|
PutVarint32(&key_size_str_, user_key_size); |
||||||
|
file_->Append(key_size_str_); |
||||||
|
offset_ += key_size_str_.length(); |
||||||
|
} |
||||||
|
|
||||||
|
// Write key
|
||||||
|
ParsedInternalKey parsed_key; |
||||||
|
if (!ParseInternalKey(key, &parsed_key)) { |
||||||
|
status_ = Status::Corruption(Slice()); |
||||||
|
return; |
||||||
|
} |
||||||
|
if (parsed_key.sequence == 0 && parsed_key.type == kTypeValue) { |
||||||
|
file_->Append(Slice(key.data(), user_key_size)); |
||||||
|
char tmp_char = PlainTableFactory::kValueTypeSeqId0; |
||||||
|
file_->Append(Slice(&tmp_char, 1)); |
||||||
|
offset_ += key.size() - 7; |
||||||
|
} else { |
||||||
|
file_->Append(key); |
||||||
|
offset_ += key.size(); |
||||||
|
} |
||||||
|
|
||||||
|
// Write value length
|
||||||
|
value_size_str_.clear(); |
||||||
|
int value_size = value.size(); |
||||||
|
PutVarint32(&value_size_str_, value_size); |
||||||
|
file_->Append(value_size_str_); |
||||||
|
|
||||||
|
// Write value
|
||||||
|
file_->Append(value); |
||||||
|
offset_ += value_size + value_size_str_.length(); |
||||||
|
|
||||||
|
properties_.num_entries++; |
||||||
|
properties_.raw_key_size += key.size(); |
||||||
|
properties_.raw_value_size += value.size(); |
||||||
|
|
||||||
|
// notify property collectors
|
||||||
|
NotifyCollectTableCollectorsOnAdd( |
||||||
|
key, |
||||||
|
value, |
||||||
|
options_.table_properties_collectors, |
||||||
|
options_.info_log.get() |
||||||
|
); |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableBuilder::status() const { return status_; } |
||||||
|
|
||||||
|
Status PlainTableBuilder::Finish() { |
||||||
|
assert(!closed_); |
||||||
|
closed_ = true; |
||||||
|
|
||||||
|
properties_.data_size = offset_; |
||||||
|
|
||||||
|
// Write the following blocks
|
||||||
|
// 1. [meta block: properties]
|
||||||
|
// 2. [metaindex block]
|
||||||
|
// 3. [footer]
|
||||||
|
MetaIndexBuilder meta_index_builer; |
||||||
|
|
||||||
|
PropertyBlockBuilder property_block_builder; |
||||||
|
// -- Add basic properties
|
||||||
|
property_block_builder.AddTableProperty(properties_); |
||||||
|
|
||||||
|
// -- Add user collected properties
|
||||||
|
NotifyCollectTableCollectorsOnFinish( |
||||||
|
options_.table_properties_collectors, |
||||||
|
options_.info_log.get(), |
||||||
|
&property_block_builder |
||||||
|
); |
||||||
|
|
||||||
|
// -- Write property block
|
||||||
|
BlockHandle property_block_handle; |
||||||
|
auto s = WriteBlock( |
||||||
|
property_block_builder.Finish(), |
||||||
|
file_, |
||||||
|
&offset_, |
||||||
|
&property_block_handle |
||||||
|
); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
meta_index_builer.Add(kPropertiesBlock, property_block_handle); |
||||||
|
|
||||||
|
// -- write metaindex block
|
||||||
|
BlockHandle metaindex_block_handle; |
||||||
|
s = WriteBlock( |
||||||
|
meta_index_builer.Finish(), |
||||||
|
file_, |
||||||
|
&offset_, |
||||||
|
&metaindex_block_handle |
||||||
|
); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
// Write Footer
|
||||||
|
Footer footer(kPlainTableMagicNumber); |
||||||
|
footer.set_metaindex_handle(metaindex_block_handle); |
||||||
|
footer.set_index_handle(BlockHandle::NullBlockHandle()); |
||||||
|
std::string footer_encoding; |
||||||
|
footer.EncodeTo(&footer_encoding); |
||||||
|
s = file_->Append(footer_encoding); |
||||||
|
if (s.ok()) { |
||||||
|
offset_ += footer_encoding.size(); |
||||||
|
} |
||||||
|
|
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableBuilder::Abandon() { |
||||||
|
closed_ = true; |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t PlainTableBuilder::NumEntries() const { |
||||||
|
return properties_.num_entries; |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t PlainTableBuilder::FileSize() const { |
||||||
|
return offset_; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,85 @@ |
|||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
//
|
||||||
|
// IndexedTable is a simple table format for UNIT TEST ONLY. It is not built
|
||||||
|
// as production quality.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <stdint.h> |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "table/table_builder.h" |
||||||
|
#include "rocksdb/table_properties.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class BlockBuilder; |
||||||
|
class BlockHandle; |
||||||
|
class WritableFile; |
||||||
|
class TableBuilder; |
||||||
|
|
||||||
|
class PlainTableBuilder: public TableBuilder { |
||||||
|
public: |
||||||
|
// Create a builder that will store the contents of the table it is
|
||||||
|
// building in *file. Does not close the file. It is up to the
|
||||||
|
// caller to close the file after calling Finish(). The output file
|
||||||
|
// will be part of level specified by 'level'. A value of -1 means
|
||||||
|
// that the caller does not know which level the output file will reside.
|
||||||
|
PlainTableBuilder(const Options& options, WritableFile* file, |
||||||
|
uint32_t user_key_size); |
||||||
|
|
||||||
|
// REQUIRES: Either Finish() or Abandon() has been called.
|
||||||
|
~PlainTableBuilder(); |
||||||
|
|
||||||
|
// Add key,value to the table being constructed.
|
||||||
|
// REQUIRES: key is after any previously added key according to comparator.
|
||||||
|
// REQUIRES: Finish(), Abandon() have not been called
|
||||||
|
void Add(const Slice& key, const Slice& value) override; |
||||||
|
|
||||||
|
// Return non-ok iff some error has been detected.
|
||||||
|
Status status() const override; |
||||||
|
|
||||||
|
// Finish building the table. Stops using the file passed to the
|
||||||
|
// constructor after this function returns.
|
||||||
|
// REQUIRES: Finish(), Abandon() have not been called
|
||||||
|
Status Finish() override; |
||||||
|
|
||||||
|
// Indicate that the contents of this builder should be abandoned. Stops
|
||||||
|
// using the file passed to the constructor after this function returns.
|
||||||
|
// If the caller is not going to call Finish(), it must call Abandon()
|
||||||
|
// before destroying this builder.
|
||||||
|
// REQUIRES: Finish(), Abandon() have not been called
|
||||||
|
void Abandon() override; |
||||||
|
|
||||||
|
// Number of calls to Add() so far.
|
||||||
|
uint64_t NumEntries() const override; |
||||||
|
|
||||||
|
// Size of the file generated so far. If invoked after a successful
|
||||||
|
// Finish() call, returns the size of the final generated file.
|
||||||
|
uint64_t FileSize() const override; |
||||||
|
|
||||||
|
private: |
||||||
|
Options options_; |
||||||
|
WritableFile* file_; |
||||||
|
uint64_t offset_ = 0; |
||||||
|
Status status_; |
||||||
|
TableProperties properties_; |
||||||
|
|
||||||
|
const size_t user_key_len_; |
||||||
|
bool closed_ = false; // Either Finish() or Abandon() has been called.
|
||||||
|
|
||||||
|
std::string key_size_str_; |
||||||
|
std::string value_size_str_; |
||||||
|
|
||||||
|
bool IsFixedLength() const { |
||||||
|
return user_key_len_ > 0; |
||||||
|
} |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
PlainTableBuilder(const PlainTableBuilder&) = delete; |
||||||
|
void operator=(const PlainTableBuilder&) = delete; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
@ -0,0 +1,40 @@ |
|||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "table/plain_table_factory.h" |
||||||
|
|
||||||
|
#include <memory> |
||||||
|
#include <stdint.h> |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "table/plain_table_builder.h" |
||||||
|
#include "table/plain_table_reader.h" |
||||||
|
#include "port/port.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
Status PlainTableFactory::NewTableReader(const Options& options, |
||||||
|
const EnvOptions& soptions, |
||||||
|
const InternalKeyComparator& icomp, |
||||||
|
unique_ptr<RandomAccessFile>&& file, |
||||||
|
uint64_t file_size, |
||||||
|
unique_ptr<TableReader>* table) const { |
||||||
|
return PlainTableReader::Open(options, soptions, icomp, std::move(file), |
||||||
|
file_size, table, bloom_bits_per_key_, |
||||||
|
hash_table_ratio_); |
||||||
|
} |
||||||
|
|
||||||
|
TableBuilder* PlainTableFactory::NewTableBuilder( |
||||||
|
const Options& options, const InternalKeyComparator& internal_comparator, |
||||||
|
WritableFile* file, CompressionType compression_type) const { |
||||||
|
return new PlainTableBuilder(options, file, user_key_len_); |
||||||
|
} |
||||||
|
|
||||||
|
extern TableFactory* NewPlainTableFactory(uint32_t user_key_len, |
||||||
|
int bloom_bits_per_key, |
||||||
|
double hash_table_ratio) { |
||||||
|
return new PlainTableFactory(user_key_len, bloom_bits_per_key, |
||||||
|
hash_table_ratio); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,76 @@ |
|||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <memory> |
||||||
|
#include <stdint.h> |
||||||
|
|
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/table.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
struct Options; |
||||||
|
struct EnvOptions; |
||||||
|
|
||||||
|
using std::unique_ptr; |
||||||
|
class Status; |
||||||
|
class RandomAccessFile; |
||||||
|
class WritableFile; |
||||||
|
class Table; |
||||||
|
class TableBuilder; |
||||||
|
|
||||||
|
// IndexedTable requires fixed length key, configured as a constructor
|
||||||
|
// parameter of the factory class. Output file format:
|
||||||
|
// +-------------+-----------------+
|
||||||
|
// | version | user_key_length |
|
||||||
|
// +------------++------------------------------+ <= key1 offset
|
||||||
|
// | [key_size] | key1 | value_size | |
|
||||||
|
// +------------+-------------+-------------+ |
|
||||||
|
// | value1 |
|
||||||
|
// | |
|
||||||
|
// +----------------------------------------+---+ <= key2 offset
|
||||||
|
// | [key_size] | key2 | value_size | |
|
||||||
|
// +------------+-------------+-------------+ |
|
||||||
|
// | value2 |
|
||||||
|
// | |
|
||||||
|
// | ...... |
|
||||||
|
// +-----------------+--------------------------+
|
||||||
|
// If user_key_length = kPlainTableVariableLength, it means the key is variable
|
||||||
|
// length, there will be an extra field for key size encoded before every key.
|
||||||
|
class PlainTableFactory : public TableFactory { |
||||||
|
public: |
||||||
|
~PlainTableFactory() {} |
||||||
|
// user_key_size is the length of the user key. If it is set to be
|
||||||
|
// kPlainTableVariableLength, then it means variable length. Otherwise, all
|
||||||
|
// the keys need to have the fix length of this value. bloom_bits_per_key is
|
||||||
|
// number of bits used for bloom filer per key. hash_table_ratio is
|
||||||
|
// the desired utilization of the hash table used for prefix hashing.
|
||||||
|
// hash_table_ratio = number of prefixes / #buckets in the hash table
|
||||||
|
explicit PlainTableFactory(uint32_t user_key_len = kPlainTableVariableLength, |
||||||
|
int bloom_bits_per_key = 0, |
||||||
|
double hash_table_ratio = 0.75) |
||||||
|
: user_key_len_(user_key_len), |
||||||
|
bloom_bits_per_key_(bloom_bits_per_key), |
||||||
|
hash_table_ratio_(hash_table_ratio) {} |
||||||
|
const char* Name() const override { return "PlainTable"; } |
||||||
|
Status NewTableReader(const Options& options, const EnvOptions& soptions, |
||||||
|
const InternalKeyComparator& internal_comparator, |
||||||
|
unique_ptr<RandomAccessFile>&& file, uint64_t file_size, |
||||||
|
unique_ptr<TableReader>* table) const override; |
||||||
|
TableBuilder* NewTableBuilder(const Options& options, |
||||||
|
const InternalKeyComparator& icomparator, |
||||||
|
WritableFile* file, |
||||||
|
CompressionType compression_type) const |
||||||
|
override; |
||||||
|
|
||||||
|
static const char kValueTypeSeqId0 = 0xFF; |
||||||
|
|
||||||
|
private: |
||||||
|
uint32_t user_key_len_; |
||||||
|
int bloom_bits_per_key_; |
||||||
|
double hash_table_ratio_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,695 @@ |
|||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "table/plain_table_reader.h" |
||||||
|
|
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "db/dbformat.h" |
||||||
|
|
||||||
|
#include "rocksdb/cache.h" |
||||||
|
#include "rocksdb/comparator.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/filter_policy.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/statistics.h" |
||||||
|
|
||||||
|
#include "table/block.h" |
||||||
|
#include "table/filter_block.h" |
||||||
|
#include "table/format.h" |
||||||
|
#include "table/meta_blocks.h" |
||||||
|
#include "table/two_level_iterator.h" |
||||||
|
#include "table/plain_table_factory.h" |
||||||
|
|
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/dynamic_bloom.h" |
||||||
|
#include "util/hash.h" |
||||||
|
#include "util/histogram.h" |
||||||
|
#include "util/murmurhash.h" |
||||||
|
#include "util/perf_context_imp.h" |
||||||
|
#include "util/stop_watch.h" |
||||||
|
|
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
inline uint32_t GetSliceHash(Slice const& s) { |
||||||
|
return Hash(s.data(), s.size(), 397) ; |
||||||
|
} |
||||||
|
|
||||||
|
inline uint32_t GetBucketIdFromHash(uint32_t hash, uint32_t num_buckets) { |
||||||
|
return hash % num_buckets; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
// Iterator to iterate IndexedTable
|
||||||
|
class PlainTableIterator : public Iterator { |
||||||
|
public: |
||||||
|
explicit PlainTableIterator(PlainTableReader* table); |
||||||
|
~PlainTableIterator(); |
||||||
|
|
||||||
|
bool Valid() const; |
||||||
|
|
||||||
|
void SeekToFirst(); |
||||||
|
|
||||||
|
void SeekToLast(); |
||||||
|
|
||||||
|
void Seek(const Slice& target); |
||||||
|
|
||||||
|
void Next(); |
||||||
|
|
||||||
|
void Prev(); |
||||||
|
|
||||||
|
Slice key() const; |
||||||
|
|
||||||
|
Slice value() const; |
||||||
|
|
||||||
|
Status status() const; |
||||||
|
|
||||||
|
private: |
||||||
|
PlainTableReader* table_; |
||||||
|
uint32_t offset_; |
||||||
|
uint32_t next_offset_; |
||||||
|
Slice key_; |
||||||
|
Slice value_; |
||||||
|
Status status_; |
||||||
|
std::string tmp_str_; |
||||||
|
// No copying allowed
|
||||||
|
PlainTableIterator(const PlainTableIterator&) = delete; |
||||||
|
void operator=(const Iterator&) = delete; |
||||||
|
}; |
||||||
|
|
||||||
|
extern const uint64_t kPlainTableMagicNumber; |
||||||
|
PlainTableReader::PlainTableReader(const EnvOptions& storage_options, |
||||||
|
const InternalKeyComparator& icomparator, |
||||||
|
uint64_t file_size, int bloom_bits_per_key, |
||||||
|
double hash_table_ratio, |
||||||
|
const TableProperties& table_properties) |
||||||
|
: soptions_(storage_options), |
||||||
|
internal_comparator_(icomparator), |
||||||
|
file_size_(file_size), |
||||||
|
kHashTableRatio(hash_table_ratio), |
||||||
|
kBloomBitsPerKey(bloom_bits_per_key), |
||||||
|
table_properties_(table_properties), |
||||||
|
data_end_offset_(table_properties_.data_size), |
||||||
|
user_key_len_(table_properties.fixed_key_len) {} |
||||||
|
|
||||||
|
PlainTableReader::~PlainTableReader() { |
||||||
|
delete[] hash_table_; |
||||||
|
delete[] sub_index_; |
||||||
|
delete bloom_; |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableReader::Open(const Options& options, |
||||||
|
const EnvOptions& soptions, |
||||||
|
const InternalKeyComparator& internal_comparator, |
||||||
|
unique_ptr<RandomAccessFile>&& file, |
||||||
|
uint64_t file_size, |
||||||
|
unique_ptr<TableReader>* table_reader, |
||||||
|
const int bloom_bits_per_key, |
||||||
|
double hash_table_ratio) { |
||||||
|
assert(options.allow_mmap_reads); |
||||||
|
|
||||||
|
if (file_size > kMaxFileSize) { |
||||||
|
return Status::NotSupported("File is too large for PlainTableReader!"); |
||||||
|
} |
||||||
|
|
||||||
|
TableProperties table_properties; |
||||||
|
auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber, |
||||||
|
options.env, options.info_log.get(), |
||||||
|
&table_properties); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
std::unique_ptr<PlainTableReader> new_reader(new PlainTableReader( |
||||||
|
soptions, internal_comparator, file_size, bloom_bits_per_key, |
||||||
|
hash_table_ratio, table_properties)); |
||||||
|
new_reader->file_ = std::move(file); |
||||||
|
new_reader->options_ = options; |
||||||
|
|
||||||
|
// -- Populate Index
|
||||||
|
s = new_reader->PopulateIndex(); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
*table_reader = std::move(new_reader); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableReader::SetupForCompaction() { |
||||||
|
} |
||||||
|
|
||||||
|
bool PlainTableReader::PrefixMayMatch(const Slice& internal_prefix) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
Iterator* PlainTableReader::NewIterator(const ReadOptions& options) { |
||||||
|
return new PlainTableIterator(this); |
||||||
|
} |
||||||
|
|
||||||
|
struct PlainTableReader::IndexRecord { |
||||||
|
uint32_t hash; // hash of the prefix
|
||||||
|
uint32_t offset; // offset of a row
|
||||||
|
IndexRecord* next; |
||||||
|
}; |
||||||
|
|
||||||
|
// Helper class to track all the index records
|
||||||
|
class PlainTableReader::IndexRecordList { |
||||||
|
public: |
||||||
|
explicit IndexRecordList(size_t num_records_per_group) |
||||||
|
: kNumRecordsPerGroup(num_records_per_group), |
||||||
|
current_group_(nullptr), |
||||||
|
num_records_in_current_group_(num_records_per_group) {} |
||||||
|
|
||||||
|
~IndexRecordList() { |
||||||
|
for (size_t i = 0; i < groups_.size(); i++) { |
||||||
|
delete[] groups_[i]; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void AddRecord(murmur_t hash, uint32_t offset) { |
||||||
|
if (num_records_in_current_group_ == kNumRecordsPerGroup) { |
||||||
|
current_group_ = AllocateNewGroup(); |
||||||
|
num_records_in_current_group_ = 0; |
||||||
|
} |
||||||
|
auto& new_record = current_group_[num_records_in_current_group_++]; |
||||||
|
new_record.hash = hash; |
||||||
|
new_record.offset = offset; |
||||||
|
new_record.next = nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
size_t GetNumRecords() const { |
||||||
|
return (groups_.size() - 1) * kNumRecordsPerGroup + |
||||||
|
num_records_in_current_group_; |
||||||
|
} |
||||||
|
IndexRecord* At(size_t index) { |
||||||
|
return &(groups_[index / kNumRecordsPerGroup][index % kNumRecordsPerGroup]); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
IndexRecord* AllocateNewGroup() { |
||||||
|
IndexRecord* result = new IndexRecord[kNumRecordsPerGroup]; |
||||||
|
groups_.push_back(result); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
const size_t kNumRecordsPerGroup; |
||||||
|
IndexRecord* current_group_; |
||||||
|
// List of arrays allocated
|
||||||
|
std::vector<IndexRecord*> groups_; |
||||||
|
size_t num_records_in_current_group_; |
||||||
|
}; |
||||||
|
|
||||||
|
int PlainTableReader::PopulateIndexRecordList(IndexRecordList* record_list) { |
||||||
|
Slice prev_key_prefix_slice; |
||||||
|
uint32_t prev_key_prefix_hash = 0; |
||||||
|
uint32_t pos = data_start_offset_; |
||||||
|
int key_index_within_prefix = 0; |
||||||
|
bool is_first_record = true; |
||||||
|
HistogramImpl keys_per_prefix_hist; |
||||||
|
// Need map to be ordered to make sure sub indexes generated
|
||||||
|
// are in order.
|
||||||
|
|
||||||
|
int num_prefixes = 0; |
||||||
|
while (pos < data_end_offset_) { |
||||||
|
uint32_t key_offset = pos; |
||||||
|
ParsedInternalKey key; |
||||||
|
Slice value_slice; |
||||||
|
status_ = Next(pos, &key, &value_slice, pos); |
||||||
|
Slice key_prefix_slice = GetPrefix(key); |
||||||
|
|
||||||
|
if (is_first_record || prev_key_prefix_slice != key_prefix_slice) { |
||||||
|
++num_prefixes; |
||||||
|
if (!is_first_record) { |
||||||
|
keys_per_prefix_hist.Add(key_index_within_prefix); |
||||||
|
} |
||||||
|
key_index_within_prefix = 0; |
||||||
|
prev_key_prefix_slice = key_prefix_slice; |
||||||
|
prev_key_prefix_hash = GetSliceHash(key_prefix_slice); |
||||||
|
} |
||||||
|
|
||||||
|
if (key_index_within_prefix++ % kIndexIntervalForSamePrefixKeys == 0) { |
||||||
|
// Add an index key for every kIndexIntervalForSamePrefixKeys keys
|
||||||
|
record_list->AddRecord(prev_key_prefix_hash, key_offset); |
||||||
|
} |
||||||
|
is_first_record = false; |
||||||
|
} |
||||||
|
|
||||||
|
keys_per_prefix_hist.Add(key_index_within_prefix); |
||||||
|
Log(options_.info_log, "Number of Keys per prefix Histogram: %s", |
||||||
|
keys_per_prefix_hist.ToString().c_str()); |
||||||
|
|
||||||
|
return num_prefixes; |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableReader::AllocateIndexAndBloom(int num_prefixes) { |
||||||
|
delete[] hash_table_; |
||||||
|
|
||||||
|
if (kBloomBitsPerKey > 0) { |
||||||
|
bloom_ = new DynamicBloom(num_prefixes * kBloomBitsPerKey); |
||||||
|
} |
||||||
|
double hash_table_size_multipier = |
||||||
|
(kHashTableRatio > 1.0) ? 1.0 : 1.0 / kHashTableRatio; |
||||||
|
hash_table_size_ = num_prefixes * hash_table_size_multipier + 1; |
||||||
|
hash_table_ = new uint32_t[hash_table_size_]; |
||||||
|
} |
||||||
|
|
||||||
|
size_t PlainTableReader::BucketizeIndexesAndFillBloom( |
||||||
|
IndexRecordList& record_list, int num_prefixes, |
||||||
|
std::vector<IndexRecord*>* hash_to_offsets, |
||||||
|
std::vector<uint32_t>* bucket_count) { |
||||||
|
size_t sub_index_size_needed = 0; |
||||||
|
bool first = true; |
||||||
|
uint32_t prev_hash = 0; |
||||||
|
size_t num_records = record_list.GetNumRecords(); |
||||||
|
for (size_t i = 0; i < num_records; i++) { |
||||||
|
IndexRecord* index_record = record_list.At(i); |
||||||
|
uint32_t cur_hash = index_record->hash; |
||||||
|
if (first || prev_hash != cur_hash) { |
||||||
|
prev_hash = cur_hash; |
||||||
|
first = false; |
||||||
|
if (bloom_) { |
||||||
|
bloom_->AddHash(cur_hash); |
||||||
|
} |
||||||
|
} |
||||||
|
uint32_t bucket = GetBucketIdFromHash(cur_hash, hash_table_size_); |
||||||
|
IndexRecord* prev_bucket_head = (*hash_to_offsets)[bucket]; |
||||||
|
index_record->next = prev_bucket_head; |
||||||
|
(*hash_to_offsets)[bucket] = index_record; |
||||||
|
auto& item_count = (*bucket_count)[bucket]; |
||||||
|
if (item_count > 0) { |
||||||
|
if (item_count == 1) { |
||||||
|
sub_index_size_needed += kOffsetLen + 1; |
||||||
|
} |
||||||
|
if (item_count == 127) { |
||||||
|
// Need more than one byte for length
|
||||||
|
sub_index_size_needed++; |
||||||
|
} |
||||||
|
sub_index_size_needed += kOffsetLen; |
||||||
|
} |
||||||
|
item_count++; |
||||||
|
} |
||||||
|
return sub_index_size_needed; |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableReader::FillIndexes( |
||||||
|
size_t sub_index_size_needed, |
||||||
|
const std::vector<IndexRecord*>& hash_to_offsets, |
||||||
|
const std::vector<uint32_t>& bucket_count) { |
||||||
|
Log(options_.info_log, "Reserving %zu bytes for sub index", |
||||||
|
sub_index_size_needed); |
||||||
|
// 8 bytes buffer for variable length size
|
||||||
|
size_t buffer_size = 8 * 8; |
||||||
|
size_t buffer_used = 0; |
||||||
|
sub_index_size_needed += buffer_size; |
||||||
|
sub_index_ = new char[sub_index_size_needed]; |
||||||
|
size_t sub_index_offset = 0; |
||||||
|
char* prev_ptr; |
||||||
|
char* cur_ptr; |
||||||
|
uint32_t* sub_index_ptr; |
||||||
|
for (int i = 0; i < hash_table_size_; i++) { |
||||||
|
uint32_t num_keys_for_bucket = bucket_count[i]; |
||||||
|
switch (num_keys_for_bucket) { |
||||||
|
case 0: |
||||||
|
// No key for bucket
|
||||||
|
hash_table_[i] = data_end_offset_; |
||||||
|
break; |
||||||
|
case 1: |
||||||
|
// point directly to the file offset
|
||||||
|
hash_table_[i] = hash_to_offsets[i]->offset; |
||||||
|
break; |
||||||
|
default: |
||||||
|
// point to second level indexes.
|
||||||
|
hash_table_[i] = sub_index_offset | kSubIndexMask; |
||||||
|
prev_ptr = sub_index_ + sub_index_offset; |
||||||
|
cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket); |
||||||
|
sub_index_offset += (cur_ptr - prev_ptr); |
||||||
|
if (cur_ptr - prev_ptr > 2 |
||||||
|
|| (cur_ptr - prev_ptr == 2 && num_keys_for_bucket <= 127)) { |
||||||
|
// Need to resize sub_index. Exponentially grow buffer.
|
||||||
|
buffer_used += cur_ptr - prev_ptr - 1; |
||||||
|
if (buffer_used + 4 > buffer_size) { |
||||||
|
Log(options_.info_log, "Recalculate suffix_map length to %zu", |
||||||
|
sub_index_size_needed); |
||||||
|
|
||||||
|
sub_index_size_needed += buffer_size; |
||||||
|
buffer_size *= 2; |
||||||
|
char* new_sub_index = new char[sub_index_size_needed]; |
||||||
|
memcpy(new_sub_index, sub_index_, sub_index_offset); |
||||||
|
delete[] sub_index_; |
||||||
|
sub_index_ = new_sub_index; |
||||||
|
} |
||||||
|
} |
||||||
|
sub_index_ptr = (uint32_t*) (sub_index_ + sub_index_offset); |
||||||
|
IndexRecord* record = hash_to_offsets[i]; |
||||||
|
int j; |
||||||
|
for (j = num_keys_for_bucket - 1; j >= 0 && record; |
||||||
|
j--, record = record->next) { |
||||||
|
sub_index_ptr[j] = record->offset; |
||||||
|
} |
||||||
|
assert(j == -1 && record == nullptr); |
||||||
|
sub_index_offset += kOffsetLen * num_keys_for_bucket; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Log(options_.info_log, "hash table size: %d, suffix_map length %zu", |
||||||
|
hash_table_size_, sub_index_size_needed); |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableReader::PopulateIndex() { |
||||||
|
// Get mmapped memory to file_data_.
|
||||||
|
Status s = file_->Read(0, file_size_, &file_data_, nullptr); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
IndexRecordList record_list(kRecordsPerGroup); |
||||||
|
// First, read the whole file, for every kIndexIntervalForSamePrefixKeys rows
|
||||||
|
// for a prefix (starting from the first one), generate a record of (hash,
|
||||||
|
// offset) and append it to IndexRecordList, which is a data structure created
|
||||||
|
// to store them.
|
||||||
|
int num_prefixes = PopulateIndexRecordList(&record_list); |
||||||
|
// Calculated hash table and bloom filter size and allocate memory for indexes
|
||||||
|
// and bloom filter based on the number of prefixes.
|
||||||
|
AllocateIndexAndBloom(num_prefixes); |
||||||
|
|
||||||
|
// Bucketize all the index records to a temp data structure, in which for
|
||||||
|
// each bucket, we generate a linked list of IndexRecord, in reversed order.
|
||||||
|
std::vector<IndexRecord*> hash_to_offsets(hash_table_size_, nullptr); |
||||||
|
std::vector<uint32_t> bucket_count(hash_table_size_, 0); |
||||||
|
size_t sub_index_size_needed = BucketizeIndexesAndFillBloom( |
||||||
|
record_list, num_prefixes, &hash_to_offsets, &bucket_count); |
||||||
|
// From the temp data structure, populate indexes.
|
||||||
|
FillIndexes(sub_index_size_needed, hash_to_offsets, bucket_count); |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableReader::GetOffset(const Slice& target, const Slice& prefix, |
||||||
|
uint32_t prefix_hash, bool& prefix_matched, |
||||||
|
uint32_t& ret_offset) { |
||||||
|
prefix_matched = false; |
||||||
|
int bucket = GetBucketIdFromHash(prefix_hash, hash_table_size_); |
||||||
|
uint32_t bucket_value = hash_table_[bucket]; |
||||||
|
if (bucket_value == data_end_offset_) { |
||||||
|
ret_offset = data_end_offset_; |
||||||
|
return Status::OK(); |
||||||
|
} else if ((bucket_value & kSubIndexMask) == 0) { |
||||||
|
// point directly to the file
|
||||||
|
ret_offset = bucket_value; |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
// point to sub-index, need to do a binary search
|
||||||
|
uint32_t low = 0; |
||||||
|
uint64_t prefix_index_offset = bucket_value ^ kSubIndexMask; |
||||||
|
|
||||||
|
const char* index_ptr = sub_index_ + prefix_index_offset; |
||||||
|
uint32_t upper_bound = 0; |
||||||
|
const uint32_t* base_ptr = (const uint32_t*) GetVarint32Ptr(index_ptr, |
||||||
|
index_ptr + 4, |
||||||
|
&upper_bound); |
||||||
|
uint32_t high = upper_bound; |
||||||
|
ParsedInternalKey mid_key; |
||||||
|
ParsedInternalKey parsed_target; |
||||||
|
if (!ParseInternalKey(target, &parsed_target)) { |
||||||
|
return Status::Corruption(Slice()); |
||||||
|
} |
||||||
|
|
||||||
|
// The key is between [low, high). Do a binary search between it.
|
||||||
|
while (high - low > 1) { |
||||||
|
uint32_t mid = (high + low) / 2; |
||||||
|
uint32_t file_offset = base_ptr[mid]; |
||||||
|
size_t tmp; |
||||||
|
Status s = ReadKey(file_data_.data() + file_offset, &mid_key, tmp); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
int cmp_result = internal_comparator_.Compare(mid_key, parsed_target); |
||||||
|
if (cmp_result < 0) { |
||||||
|
low = mid; |
||||||
|
} else { |
||||||
|
if (cmp_result == 0) { |
||||||
|
// Happen to have found the exact key or target is smaller than the
|
||||||
|
// first key after base_offset.
|
||||||
|
prefix_matched = true; |
||||||
|
ret_offset = file_offset; |
||||||
|
return Status::OK(); |
||||||
|
} else { |
||||||
|
high = mid; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
// Both of the key at the position low or low+1 could share the same
|
||||||
|
// prefix as target. We need to rule out one of them to avoid to go
|
||||||
|
// to the wrong prefix.
|
||||||
|
ParsedInternalKey low_key; |
||||||
|
size_t tmp; |
||||||
|
uint32_t low_key_offset = base_ptr[low]; |
||||||
|
Status s = ReadKey(file_data_.data() + low_key_offset, &low_key, tmp); |
||||||
|
if (GetPrefix(low_key) == prefix) { |
||||||
|
prefix_matched = true; |
||||||
|
ret_offset = low_key_offset; |
||||||
|
} else if (low + 1 < upper_bound) { |
||||||
|
// There is possible a next prefix, return it
|
||||||
|
prefix_matched = false; |
||||||
|
ret_offset = base_ptr[low + 1]; |
||||||
|
} else { |
||||||
|
// target is larger than a key of the last prefix in this bucket
|
||||||
|
// but with a different prefix. Key does not exist.
|
||||||
|
ret_offset = data_end_offset_; |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
bool PlainTableReader::MayHavePrefix(uint32_t hash) { |
||||||
|
return bloom_ == nullptr || bloom_->MayContainHash(hash); |
||||||
|
} |
||||||
|
|
||||||
|
Slice PlainTableReader::GetPrefix(const ParsedInternalKey& target) { |
||||||
|
return options_.prefix_extractor->Transform(target.user_key); |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableReader::ReadKey(const char* row_ptr, ParsedInternalKey* key, |
||||||
|
size_t& bytes_read) { |
||||||
|
const char* key_ptr = nullptr; |
||||||
|
bytes_read = 0; |
||||||
|
size_t user_key_size = 0; |
||||||
|
if (IsFixedLength()) { |
||||||
|
user_key_size = user_key_len_; |
||||||
|
key_ptr = row_ptr; |
||||||
|
} else { |
||||||
|
uint32_t tmp_size = 0; |
||||||
|
key_ptr = GetVarint32Ptr(row_ptr, file_data_.data() + data_end_offset_, |
||||||
|
&tmp_size); |
||||||
|
if (key_ptr == nullptr) { |
||||||
|
return Status::Corruption("Unable to read the next key"); |
||||||
|
} |
||||||
|
user_key_size = (size_t)tmp_size; |
||||||
|
bytes_read = key_ptr - row_ptr; |
||||||
|
} |
||||||
|
if (key_ptr + user_key_size + 1 >= file_data_.data() + data_end_offset_) { |
||||||
|
return Status::Corruption("Unable to read the next key"); |
||||||
|
} |
||||||
|
|
||||||
|
if (*(key_ptr + user_key_size) == PlainTableFactory::kValueTypeSeqId0) { |
||||||
|
// Special encoding for the row with seqID=0
|
||||||
|
key->user_key = Slice(key_ptr, user_key_size); |
||||||
|
key->sequence = 0; |
||||||
|
key->type = kTypeValue; |
||||||
|
bytes_read += user_key_size + 1; |
||||||
|
} else { |
||||||
|
if (row_ptr + user_key_size + 8 >= file_data_.data() + data_end_offset_) { |
||||||
|
return Status::Corruption("Unable to read the next key"); |
||||||
|
} |
||||||
|
if (!ParseInternalKey(Slice(key_ptr, user_key_size + 8), key)) { |
||||||
|
return Status::Corruption(Slice()); |
||||||
|
} |
||||||
|
bytes_read += user_key_size + 8; |
||||||
|
} |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableReader::Next(uint32_t offset, ParsedInternalKey* key, |
||||||
|
Slice* value, uint32_t& next_offset) { |
||||||
|
if (offset == data_end_offset_) { |
||||||
|
next_offset = data_end_offset_; |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
if (offset > data_end_offset_) { |
||||||
|
return Status::Corruption("Offset is out of file size"); |
||||||
|
} |
||||||
|
|
||||||
|
const char* row_ptr = file_data_.data() + offset; |
||||||
|
size_t bytes_for_key; |
||||||
|
Status s = ReadKey(row_ptr, key, bytes_for_key); |
||||||
|
uint32_t value_size; |
||||||
|
const char* value_ptr = GetVarint32Ptr(row_ptr + bytes_for_key, |
||||||
|
file_data_.data() + data_end_offset_, |
||||||
|
&value_size); |
||||||
|
if (value_ptr == nullptr) { |
||||||
|
return Status::Corruption("Error reading value length."); |
||||||
|
} |
||||||
|
next_offset = offset + (value_ptr - row_ptr) + value_size; |
||||||
|
if (next_offset > data_end_offset_) { |
||||||
|
return Status::Corruption("Reach end of file when reading value"); |
||||||
|
} |
||||||
|
*value = Slice(value_ptr, value_size); |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableReader::Get(const ReadOptions& ro, const Slice& target, |
||||||
|
void* arg, |
||||||
|
bool (*saver)(void*, const ParsedInternalKey&, |
||||||
|
const Slice&, bool), |
||||||
|
void (*mark_key_may_exist)(void*)) { |
||||||
|
// Check bloom filter first.
|
||||||
|
Slice prefix_slice = GetPrefix(target); |
||||||
|
uint32_t prefix_hash = GetSliceHash(prefix_slice); |
||||||
|
if (!MayHavePrefix(prefix_hash)) { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
uint32_t offset; |
||||||
|
bool prefix_match; |
||||||
|
Status s = GetOffset(target, prefix_slice, prefix_hash, prefix_match, offset); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
ParsedInternalKey found_key; |
||||||
|
ParsedInternalKey parsed_target; |
||||||
|
if (!ParseInternalKey(target, &parsed_target)) { |
||||||
|
return Status::Corruption(Slice()); |
||||||
|
} |
||||||
|
|
||||||
|
Slice found_value; |
||||||
|
while (offset < data_end_offset_) { |
||||||
|
Status s = Next(offset, &found_key, &found_value, offset); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
if (!prefix_match) { |
||||||
|
// Need to verify prefix for the first key found if it is not yet
|
||||||
|
// checked.
|
||||||
|
if (GetPrefix(found_key) != prefix_slice) { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
prefix_match = true; |
||||||
|
} |
||||||
|
if (internal_comparator_.Compare(found_key, parsed_target) >= 0) { |
||||||
|
if (!(*saver)(arg, found_key, found_value, true)) { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t PlainTableReader::ApproximateOffsetOf(const Slice& key) { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
PlainTableIterator::PlainTableIterator(PlainTableReader* table) : |
||||||
|
table_(table) { |
||||||
|
next_offset_ = offset_ = table_->data_end_offset_; |
||||||
|
} |
||||||
|
|
||||||
|
PlainTableIterator::~PlainTableIterator() { |
||||||
|
} |
||||||
|
|
||||||
|
bool PlainTableIterator::Valid() const { |
||||||
|
return offset_ < table_->data_end_offset_ |
||||||
|
&& offset_ >= table_->data_start_offset_; |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableIterator::SeekToFirst() { |
||||||
|
next_offset_ = table_->data_start_offset_; |
||||||
|
if (next_offset_ >= table_->data_end_offset_) { |
||||||
|
next_offset_ = offset_ = table_->data_end_offset_; |
||||||
|
} else { |
||||||
|
Next(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableIterator::SeekToLast() { |
||||||
|
assert(false); |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableIterator::Seek(const Slice& target) { |
||||||
|
Slice prefix_slice = table_->GetPrefix(target); |
||||||
|
uint32_t prefix_hash = GetSliceHash(prefix_slice); |
||||||
|
if (!table_->MayHavePrefix(prefix_hash)) { |
||||||
|
offset_ = next_offset_ = table_->data_end_offset_; |
||||||
|
return; |
||||||
|
} |
||||||
|
bool prefix_match; |
||||||
|
status_ = table_->GetOffset(target, prefix_slice, prefix_hash, prefix_match, |
||||||
|
next_offset_); |
||||||
|
if (!status_.ok()) { |
||||||
|
offset_ = next_offset_ = table_->data_end_offset_; |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
if (next_offset_ < table_-> data_end_offset_) { |
||||||
|
for (Next(); status_.ok() && Valid(); Next()) { |
||||||
|
if (!prefix_match) { |
||||||
|
// Need to verify the first key's prefix
|
||||||
|
if (table_->GetPrefix(key()) != prefix_slice) { |
||||||
|
offset_ = next_offset_ = table_->data_end_offset_; |
||||||
|
break; |
||||||
|
} |
||||||
|
prefix_match = true; |
||||||
|
} |
||||||
|
if (table_->internal_comparator_.Compare(key(), target) >= 0) { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} else { |
||||||
|
offset_ = table_->data_end_offset_; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableIterator::Next() { |
||||||
|
offset_ = next_offset_; |
||||||
|
if (offset_ < table_->data_end_offset_) { |
||||||
|
Slice tmp_slice; |
||||||
|
ParsedInternalKey parsed_key; |
||||||
|
status_ = table_->Next(next_offset_, &parsed_key, &value_, next_offset_); |
||||||
|
if (status_.ok()) { |
||||||
|
// Make a copy in this case. TODO optimize.
|
||||||
|
tmp_str_.clear(); |
||||||
|
AppendInternalKey(&tmp_str_, parsed_key); |
||||||
|
key_ = Slice(tmp_str_); |
||||||
|
} else { |
||||||
|
offset_ = next_offset_ = table_->data_end_offset_; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void PlainTableIterator::Prev() { |
||||||
|
assert(false); |
||||||
|
} |
||||||
|
|
||||||
|
Slice PlainTableIterator::key() const { |
||||||
|
assert(Valid()); |
||||||
|
return key_; |
||||||
|
} |
||||||
|
|
||||||
|
Slice PlainTableIterator::value() const { |
||||||
|
assert(Valid()); |
||||||
|
return value_; |
||||||
|
} |
||||||
|
|
||||||
|
Status PlainTableIterator::status() const { |
||||||
|
return status_; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,220 @@ |
|||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <unordered_map> |
||||||
|
#include <memory> |
||||||
|
#include <vector> |
||||||
|
#include <string> |
||||||
|
#include <stdint.h> |
||||||
|
|
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/iterator.h" |
||||||
|
#include "rocksdb/slice_transform.h" |
||||||
|
#include "rocksdb/table.h" |
||||||
|
#include "rocksdb/table_properties.h" |
||||||
|
#include "table/table_reader.h" |
||||||
|
#include "table/plain_table_factory.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Block; |
||||||
|
class BlockHandle; |
||||||
|
class Footer; |
||||||
|
struct Options; |
||||||
|
class RandomAccessFile; |
||||||
|
struct ReadOptions; |
||||||
|
class TableCache; |
||||||
|
class TableReader; |
||||||
|
class DynamicBloom; |
||||||
|
class InternalKeyComparator; |
||||||
|
|
||||||
|
using std::unique_ptr; |
||||||
|
using std::unordered_map; |
||||||
|
extern const uint32_t kPlainTableVariableLength; |
||||||
|
|
||||||
|
// Based on following output file format shown in plain_table_factory.h
|
||||||
|
// When opening the output file, IndexedTableReader creates a hash table
|
||||||
|
// from key prefixes to offset of the output file. IndexedTable will decide
|
||||||
|
// whether it points to the data offset of the first key with the key prefix
|
||||||
|
// or the offset of it. If there are too many keys share this prefix, it will
|
||||||
|
// create a binary search-able index from the suffix to offset on disk.
|
||||||
|
//
|
||||||
|
// The implementation of IndexedTableReader requires output file is mmaped
|
||||||
|
class PlainTableReader: public TableReader { |
||||||
|
public: |
||||||
|
static Status Open(const Options& options, const EnvOptions& soptions, |
||||||
|
const InternalKeyComparator& internal_comparator, |
||||||
|
unique_ptr<RandomAccessFile>&& file, uint64_t file_size, |
||||||
|
unique_ptr<TableReader>* table, |
||||||
|
const int bloom_bits_per_key, double hash_table_ratio); |
||||||
|
|
||||||
|
bool PrefixMayMatch(const Slice& internal_prefix); |
||||||
|
|
||||||
|
Iterator* NewIterator(const ReadOptions&); |
||||||
|
|
||||||
|
Status Get(const ReadOptions&, const Slice& key, void* arg, |
||||||
|
bool (*result_handler)(void* arg, const ParsedInternalKey& k, |
||||||
|
const Slice& v, bool), |
||||||
|
void (*mark_key_may_exist)(void*) = nullptr); |
||||||
|
|
||||||
|
uint64_t ApproximateOffsetOf(const Slice& key); |
||||||
|
|
||||||
|
void SetupForCompaction(); |
||||||
|
|
||||||
|
const TableProperties& GetTableProperties() { return table_properties_; } |
||||||
|
|
||||||
|
PlainTableReader(const EnvOptions& storage_options, |
||||||
|
const InternalKeyComparator& internal_comparator, |
||||||
|
uint64_t file_size, int bloom_num_bits, |
||||||
|
double hash_table_ratio, |
||||||
|
const TableProperties& table_properties); |
||||||
|
~PlainTableReader(); |
||||||
|
|
||||||
|
private: |
||||||
|
struct IndexRecord; |
||||||
|
class IndexRecordList; |
||||||
|
|
||||||
|
uint32_t* hash_table_ = nullptr; |
||||||
|
int hash_table_size_ = 0; |
||||||
|
char* sub_index_ = nullptr; |
||||||
|
|
||||||
|
Options options_; |
||||||
|
const EnvOptions& soptions_; |
||||||
|
const InternalKeyComparator internal_comparator_; |
||||||
|
Status status_; |
||||||
|
unique_ptr<RandomAccessFile> file_; |
||||||
|
|
||||||
|
Slice file_data_; |
||||||
|
uint32_t version_; |
||||||
|
uint32_t file_size_; |
||||||
|
|
||||||
|
const double kHashTableRatio; |
||||||
|
const int kBloomBitsPerKey; |
||||||
|
DynamicBloom* bloom_ = nullptr; |
||||||
|
|
||||||
|
TableProperties table_properties_; |
||||||
|
const uint32_t data_start_offset_ = 0; |
||||||
|
const uint32_t data_end_offset_; |
||||||
|
const size_t user_key_len_; |
||||||
|
|
||||||
|
static const size_t kNumInternalBytes = 8; |
||||||
|
static const uint32_t kSubIndexMask = 0x80000000; |
||||||
|
static const size_t kOffsetLen = sizeof(uint32_t); |
||||||
|
static const uint64_t kMaxFileSize = 1u << 31; |
||||||
|
static const size_t kRecordsPerGroup = 256; |
||||||
|
// To speed up the search for keys with same prefix, we'll add index key for
|
||||||
|
// every N keys, where the "N" is determined by
|
||||||
|
// kIndexIntervalForSamePrefixKeys
|
||||||
|
static const size_t kIndexIntervalForSamePrefixKeys = 16; |
||||||
|
|
||||||
|
bool IsFixedLength() const { |
||||||
|
return user_key_len_ != kPlainTableVariableLength; |
||||||
|
} |
||||||
|
|
||||||
|
size_t GetFixedInternalKeyLength() const { |
||||||
|
return user_key_len_ + kNumInternalBytes; |
||||||
|
} |
||||||
|
|
||||||
|
friend class TableCache; |
||||||
|
friend class PlainTableIterator; |
||||||
|
|
||||||
|
// Internal helper function to generate an IndexRecordList object from all
|
||||||
|
// the rows, which contains index records as a list.
|
||||||
|
int PopulateIndexRecordList(IndexRecordList* record_list); |
||||||
|
|
||||||
|
// Internal helper function to allocate memory for indexes and bloom filters
|
||||||
|
void AllocateIndexAndBloom(int num_prefixes); |
||||||
|
|
||||||
|
// Internal helper function to bucket index record list to hash buckets.
|
||||||
|
// hash_to_offsets is sized of of hash_table_size_, each contains a linked
|
||||||
|
// list
|
||||||
|
// of offsets for the hash, in reversed order.
|
||||||
|
// bucket_count is sized of hash_table_size_. The value is how many index
|
||||||
|
// records are there in hash_to_offsets for the same bucket.
|
||||||
|
size_t BucketizeIndexesAndFillBloom( |
||||||
|
IndexRecordList& record_list, int num_prefixes, |
||||||
|
std::vector<IndexRecord*>* hash_to_offsets, |
||||||
|
std::vector<uint32_t>* bucket_count); |
||||||
|
|
||||||
|
// Internal helper class to fill the indexes and bloom filters to internal
|
||||||
|
// data structures. hash_to_offsets and bucket_count are bucketized indexes
|
||||||
|
// and counts generated by BucketizeIndexesAndFillBloom().
|
||||||
|
void FillIndexes(size_t sub_index_size_needed, |
||||||
|
const std::vector<IndexRecord*>& hash_to_offsets, |
||||||
|
const std::vector<uint32_t>& bucket_count); |
||||||
|
|
||||||
|
// PopulateIndex() builds index of keys. It must be called before any query
|
||||||
|
// to the table.
|
||||||
|
//
|
||||||
|
// hash_table_ contains buckets size of hash_table_size_, each is a 32-bit
|
||||||
|
// integer. The lower 31 bits contain an offset value (explained below) and
|
||||||
|
// the first bit of the integer indicates type of the offset.
|
||||||
|
//
|
||||||
|
// +--------------+------------------------------------------------------+
|
||||||
|
// | Flag (1 bit) | Offset to binary search buffer or file (31 bits) +
|
||||||
|
// +--------------+------------------------------------------------------+
|
||||||
|
//
|
||||||
|
// Explanation for the "flag bit":
|
||||||
|
//
|
||||||
|
// 0 indicates that the bucket contains only one prefix (no conflict when
|
||||||
|
// hashing this prefix), whose first row starts from this offset of the
|
||||||
|
// file.
|
||||||
|
// 1 indicates that the bucket contains more than one prefixes, or there
|
||||||
|
// are too many rows for one prefix so we need a binary search for it. In
|
||||||
|
// this case, the offset indicates the offset of sub_index_ holding the
|
||||||
|
// binary search indexes of keys for those rows. Those binary search indexes
|
||||||
|
// are organized in this way:
|
||||||
|
//
|
||||||
|
// The first 4 bytes, indicate how many indexes (N) are stored after it. After
|
||||||
|
// it, there are N 32-bit integers, each points of an offset of the file,
|
||||||
|
// which
|
||||||
|
// points to starting of a row. Those offsets need to be guaranteed to be in
|
||||||
|
// ascending order so the keys they are pointing to are also in ascending
|
||||||
|
// order
|
||||||
|
// to make sure we can use them to do binary searches. Below is visual
|
||||||
|
// presentation of a bucket.
|
||||||
|
//
|
||||||
|
// <begin>
|
||||||
|
// number_of_records: varint32
|
||||||
|
// record 1 file offset: fixedint32
|
||||||
|
// record 2 file offset: fixedint32
|
||||||
|
// ....
|
||||||
|
// record N file offset: fixedint32
|
||||||
|
// <end>
|
||||||
|
Status PopulateIndex(); |
||||||
|
|
||||||
|
// Check bloom filter to see whether it might contain this prefix.
|
||||||
|
// The hash of the prefix is given, since it can be reused for index lookup
|
||||||
|
// too.
|
||||||
|
bool MayHavePrefix(uint32_t hash); |
||||||
|
|
||||||
|
Status ReadKey(const char* row_ptr, ParsedInternalKey* key, |
||||||
|
size_t& bytes_read); |
||||||
|
// Read the key and value at offset to key and value.
|
||||||
|
// tmp_slice is a tmp slice.
|
||||||
|
// return next_offset as the offset for the next key.
|
||||||
|
Status Next(uint32_t offset, ParsedInternalKey* key, Slice* value, |
||||||
|
uint32_t& next_offset); |
||||||
|
// Get file offset for key target.
|
||||||
|
// return value prefix_matched is set to true if the offset is confirmed
|
||||||
|
// for a key with the same prefix as target.
|
||||||
|
Status GetOffset(const Slice& target, const Slice& prefix, |
||||||
|
uint32_t prefix_hash, bool& prefix_matched, |
||||||
|
uint32_t& ret_offset); |
||||||
|
|
||||||
|
Slice GetPrefix(const Slice& target) { |
||||||
|
assert(target.size() >= 8); // target is internal key
|
||||||
|
return options_.prefix_extractor->Transform( |
||||||
|
Slice(target.data(), target.size() - 8)); |
||||||
|
} |
||||||
|
|
||||||
|
Slice GetPrefix(const ParsedInternalKey& target); |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
explicit PlainTableReader(const TableReader&) = delete; |
||||||
|
void operator=(const TableReader&) = delete; |
||||||
|
}; |
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,55 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Slice; |
||||||
|
class Status; |
||||||
|
|
||||||
|
// TableBuilder provides the interface used to build a Table
|
||||||
|
// (an immutable and sorted map from keys to values).
|
||||||
|
//
|
||||||
|
// Multiple threads can invoke const methods on a TableBuilder without
|
||||||
|
// external synchronization, but if any of the threads may call a
|
||||||
|
// non-const method, all threads accessing the same TableBuilder must use
|
||||||
|
// external synchronization.
|
||||||
|
class TableBuilder { |
||||||
|
public: |
||||||
|
// REQUIRES: Either Finish() or Abandon() has been called.
|
||||||
|
virtual ~TableBuilder() {} |
||||||
|
|
||||||
|
// Add key,value to the table being constructed.
|
||||||
|
// REQUIRES: key is after any previously added key according to comparator.
|
||||||
|
// REQUIRES: Finish(), Abandon() have not been called
|
||||||
|
virtual void Add(const Slice& key, const Slice& value) = 0; |
||||||
|
|
||||||
|
// Return non-ok iff some error has been detected.
|
||||||
|
virtual Status status() const = 0; |
||||||
|
|
||||||
|
// Finish building the table.
|
||||||
|
// REQUIRES: Finish(), Abandon() have not been called
|
||||||
|
virtual Status Finish() = 0; |
||||||
|
|
||||||
|
// Indicate that the contents of this builder should be abandoned.
|
||||||
|
// If the caller is not going to call Finish(), it must call Abandon()
|
||||||
|
// before destroying this builder.
|
||||||
|
// REQUIRES: Finish(), Abandon() have not been called
|
||||||
|
virtual void Abandon() = 0; |
||||||
|
|
||||||
|
// Number of calls to Add() so far.
|
||||||
|
virtual uint64_t NumEntries() const = 0; |
||||||
|
|
||||||
|
// Size of the file generated so far. If invoked after a successful
|
||||||
|
// Finish() call, returns the size of the final generated file.
|
||||||
|
virtual uint64_t FileSize() const = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,114 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "rocksdb/table_properties.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace { |
||||||
|
void AppendProperty( |
||||||
|
std::string& props, |
||||||
|
const std::string& key, |
||||||
|
const std::string& value, |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) { |
||||||
|
props.append(key); |
||||||
|
props.append(kv_delim); |
||||||
|
props.append(value); |
||||||
|
props.append(prop_delim); |
||||||
|
} |
||||||
|
|
||||||
|
template <class TValue> |
||||||
|
void AppendProperty( |
||||||
|
std::string& props, |
||||||
|
const std::string& key, |
||||||
|
const TValue& value, |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) { |
||||||
|
AppendProperty( |
||||||
|
props, key, std::to_string(value), prop_delim, kv_delim |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
std::string TableProperties::ToString( |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) const { |
||||||
|
std::string result; |
||||||
|
result.reserve(1024); |
||||||
|
|
||||||
|
// Basic Info
|
||||||
|
AppendProperty( |
||||||
|
result, "# data blocks", num_data_blocks, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); |
||||||
|
|
||||||
|
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"raw average key size", |
||||||
|
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, "raw value size", raw_value_size, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"raw average value size", |
||||||
|
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); |
||||||
|
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); |
||||||
|
AppendProperty( |
||||||
|
result, "filter block size", filter_size, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"(estimated) table size", |
||||||
|
data_size + index_size + filter_size, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"filter policy name", |
||||||
|
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
const std::string TablePropertiesNames::kDataSize = |
||||||
|
"rocksdb.data.size"; |
||||||
|
const std::string TablePropertiesNames::kIndexSize = |
||||||
|
"rocksdb.index.size"; |
||||||
|
const std::string TablePropertiesNames::kFilterSize = |
||||||
|
"rocksdb.filter.size"; |
||||||
|
const std::string TablePropertiesNames::kRawKeySize = |
||||||
|
"rocksdb.raw.key.size"; |
||||||
|
const std::string TablePropertiesNames::kRawValueSize = |
||||||
|
"rocksdb.raw.value.size"; |
||||||
|
const std::string TablePropertiesNames::kNumDataBlocks = |
||||||
|
"rocksdb.num.data.blocks"; |
||||||
|
const std::string TablePropertiesNames::kNumEntries = |
||||||
|
"rocksdb.num.entries"; |
||||||
|
const std::string TablePropertiesNames::kFilterPolicy = |
||||||
|
"rocksdb.filter.policy"; |
||||||
|
const std::string TablePropertiesNames::kFormatVersion = |
||||||
|
"rocksdb.format.version"; |
||||||
|
const std::string TablePropertiesNames::kFixedKeyLen = |
||||||
|
"rocksdb.fixed.key.length"; |
||||||
|
|
||||||
|
extern const std::string kPropertiesBlock = "rocksdb.properties"; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,71 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Iterator; |
||||||
|
struct ParsedInternalKey; |
||||||
|
class Slice; |
||||||
|
struct ReadOptions; |
||||||
|
struct TableProperties; |
||||||
|
|
||||||
|
// A Table is a sorted map from strings to strings. Tables are
|
||||||
|
// immutable and persistent. A Table may be safely accessed from
|
||||||
|
// multiple threads without external synchronization.
|
||||||
|
class TableReader { |
||||||
|
public: |
||||||
|
virtual ~TableReader() {} |
||||||
|
|
||||||
|
// Determine whether there is a chance that the current table file
|
||||||
|
// contains the key a key starting with iternal_prefix. The specific
|
||||||
|
// table implementation can use bloom filter and/or other heuristic
|
||||||
|
// to filter out this table as a whole.
|
||||||
|
virtual bool PrefixMayMatch(const Slice& internal_prefix) = 0; |
||||||
|
|
||||||
|
// Returns a new iterator over the table contents.
|
||||||
|
// The result of NewIterator() is initially invalid (caller must
|
||||||
|
// call one of the Seek methods on the iterator before using it).
|
||||||
|
virtual Iterator* NewIterator(const ReadOptions&) = 0; |
||||||
|
|
||||||
|
// Given a key, return an approximate byte offset in the file where
|
||||||
|
// the data for that key begins (or would begin if the key were
|
||||||
|
// present in the file). The returned value is in terms of file
|
||||||
|
// bytes, and so includes effects like compression of the underlying data.
|
||||||
|
// E.g., the approximate offset of the last key in the table will
|
||||||
|
// be close to the file length.
|
||||||
|
virtual uint64_t ApproximateOffsetOf(const Slice& key) = 0; |
||||||
|
|
||||||
|
// Set up the table for Compaction. Might change some parameters with
|
||||||
|
// posix_fadvise
|
||||||
|
virtual void SetupForCompaction() = 0; |
||||||
|
|
||||||
|
virtual const TableProperties& GetTableProperties() = 0; |
||||||
|
|
||||||
|
// Calls (*result_handler)(handle_context, ...) repeatedly, starting with
|
||||||
|
// the entry found after a call to Seek(key), until result_handler returns
|
||||||
|
// false, where k is the actual internal key for a row found and v as the
|
||||||
|
// value of the key. didIO is true if I/O is involved in the operation. May
|
||||||
|
// not make such a call if filter policy says that key is not present.
|
||||||
|
//
|
||||||
|
// mark_key_may_exist_handler needs to be called when it is configured to be
|
||||||
|
// memory only and the key is not found in the block cache, with
|
||||||
|
// the parameter to be handle_context.
|
||||||
|
//
|
||||||
|
// readOptions is the options for the read
|
||||||
|
// key is the key to search for
|
||||||
|
virtual Status Get( |
||||||
|
const ReadOptions& readOptions, const Slice& key, void* handle_context, |
||||||
|
bool (*result_handler)(void* arg, const ParsedInternalKey& k, |
||||||
|
const Slice& v, bool didIO), |
||||||
|
void (*mark_key_may_exist_handler)(void* handle_context) = nullptr) = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,36 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "dynamic_bloom.h" |
||||||
|
|
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "util/hash.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace { |
||||||
|
static uint32_t BloomHash(const Slice& key) { |
||||||
|
return Hash(key.data(), key.size(), 0xbc9f1d34); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
DynamicBloom::DynamicBloom(uint32_t total_bits, |
||||||
|
uint32_t (*hash_func)(const Slice& key), |
||||||
|
uint32_t num_probes) |
||||||
|
: hash_func_(hash_func), |
||||||
|
kTotalBits((total_bits + 7) / 8 * 8), |
||||||
|
kNumProbes(num_probes) { |
||||||
|
assert(hash_func_); |
||||||
|
assert(kNumProbes > 0); |
||||||
|
assert(kTotalBits > 0); |
||||||
|
data_.reset(new unsigned char[kTotalBits / 8]()); |
||||||
|
} |
||||||
|
|
||||||
|
DynamicBloom::DynamicBloom(uint32_t total_bits, |
||||||
|
uint32_t num_probes) |
||||||
|
: DynamicBloom(total_bits, &BloomHash, num_probes) { |
||||||
|
} |
||||||
|
|
||||||
|
} // rocksdb
|
||||||
@ -0,0 +1,72 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <atomic> |
||||||
|
#include <memory> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Slice; |
||||||
|
|
||||||
|
class DynamicBloom { |
||||||
|
public: |
||||||
|
// total_bits: fixed total bits for the bloom
|
||||||
|
// hash_func: customized hash function
|
||||||
|
// num_probes: number of hash probes for a single key
|
||||||
|
DynamicBloom(uint32_t total_bits, |
||||||
|
uint32_t (*hash_func)(const Slice& key), |
||||||
|
uint32_t num_probes = 6); |
||||||
|
|
||||||
|
explicit DynamicBloom(uint32_t total_bits, uint32_t num_probes = 6); |
||||||
|
|
||||||
|
// Assuming single threaded access to this function.
|
||||||
|
void Add(const Slice& key); |
||||||
|
|
||||||
|
// Assuming single threaded access to this function.
|
||||||
|
void AddHash(uint32_t hash); |
||||||
|
|
||||||
|
// Multithreaded access to this function is OK
|
||||||
|
bool MayContain(const Slice& key); |
||||||
|
|
||||||
|
// Multithreaded access to this function is OK
|
||||||
|
bool MayContainHash(uint32_t hash); |
||||||
|
|
||||||
|
private: |
||||||
|
uint32_t (*hash_func_)(const Slice& key); |
||||||
|
const uint32_t kTotalBits; |
||||||
|
const uint32_t kNumProbes; |
||||||
|
std::unique_ptr<unsigned char[]> data_; |
||||||
|
}; |
||||||
|
|
||||||
|
inline void DynamicBloom::Add(const Slice& key) { AddHash(hash_func_(key)); } |
||||||
|
|
||||||
|
inline bool DynamicBloom::MayContain(const Slice& key) { |
||||||
|
return (MayContainHash(hash_func_(key))); |
||||||
|
} |
||||||
|
|
||||||
|
inline bool DynamicBloom::MayContainHash(uint32_t h) { |
||||||
|
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
|
||||||
|
for (uint32_t i = 0; i < kNumProbes; i++) { |
||||||
|
const uint32_t bitpos = h % kTotalBits; |
||||||
|
if (((data_[bitpos / 8]) & (1 << (bitpos % 8))) == 0) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
h += delta; |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
inline void DynamicBloom::AddHash(uint32_t h) { |
||||||
|
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
|
||||||
|
for (uint32_t i = 0; i < kNumProbes; i++) { |
||||||
|
const uint32_t bitpos = h % kTotalBits; |
||||||
|
data_[bitpos / 8] |= (1 << (bitpos % 8)); |
||||||
|
h += delta; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // rocksdb
|
||||||
@ -0,0 +1,113 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include <gflags/gflags.h> |
||||||
|
|
||||||
|
#include "dynamic_bloom.h" |
||||||
|
#include "util/logging.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
|
||||||
|
DEFINE_int32(bits_per_key, 10, ""); |
||||||
|
DEFINE_int32(num_probes, 6, ""); |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
static Slice Key(int i, char* buffer) { |
||||||
|
memcpy(buffer, &i, sizeof(i)); |
||||||
|
return Slice(buffer, sizeof(i)); |
||||||
|
} |
||||||
|
|
||||||
|
class DynamicBloomTest { |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(DynamicBloomTest, EmptyFilter) { |
||||||
|
DynamicBloom bloom(100, 2); |
||||||
|
ASSERT_TRUE(! bloom.MayContain("hello")); |
||||||
|
ASSERT_TRUE(! bloom.MayContain("world")); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(DynamicBloomTest, Small) { |
||||||
|
DynamicBloom bloom(100, 2); |
||||||
|
bloom.Add("hello"); |
||||||
|
bloom.Add("world"); |
||||||
|
ASSERT_TRUE(bloom.MayContain("hello")); |
||||||
|
ASSERT_TRUE(bloom.MayContain("world")); |
||||||
|
ASSERT_TRUE(! bloom.MayContain("x")); |
||||||
|
ASSERT_TRUE(! bloom.MayContain("foo")); |
||||||
|
} |
||||||
|
|
||||||
|
static int NextLength(int length) { |
||||||
|
if (length < 10) { |
||||||
|
length += 1; |
||||||
|
} else if (length < 100) { |
||||||
|
length += 10; |
||||||
|
} else if (length < 1000) { |
||||||
|
length += 100; |
||||||
|
} else { |
||||||
|
length += 1000; |
||||||
|
} |
||||||
|
return length; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(DynamicBloomTest, VaryingLengths) { |
||||||
|
char buffer[sizeof(int)]; |
||||||
|
|
||||||
|
// Count number of filters that significantly exceed the false positive rate
|
||||||
|
int mediocre_filters = 0; |
||||||
|
int good_filters = 0; |
||||||
|
|
||||||
|
fprintf(stderr, "bits_per_key: %d num_probes: %d\n", |
||||||
|
FLAGS_bits_per_key, FLAGS_num_probes); |
||||||
|
|
||||||
|
for (int length = 1; length <= 10000; length = NextLength(length)) { |
||||||
|
uint32_t bloom_bits = std::max(length * FLAGS_bits_per_key, 64); |
||||||
|
DynamicBloom bloom(bloom_bits, FLAGS_num_probes); |
||||||
|
for (int i = 0; i < length; i++) { |
||||||
|
bloom.Add(Key(i, buffer)); |
||||||
|
ASSERT_TRUE(bloom.MayContain(Key(i, buffer))); |
||||||
|
} |
||||||
|
|
||||||
|
// All added keys must match
|
||||||
|
for (int i = 0; i < length; i++) { |
||||||
|
ASSERT_TRUE(bloom.MayContain(Key(i, buffer))) |
||||||
|
<< "Length " << length << "; key " << i; |
||||||
|
} |
||||||
|
|
||||||
|
// Check false positive rate
|
||||||
|
|
||||||
|
int result = 0; |
||||||
|
for (int i = 0; i < 10000; i++) { |
||||||
|
if (bloom.MayContain(Key(i + 1000000000, buffer))) { |
||||||
|
result++; |
||||||
|
} |
||||||
|
} |
||||||
|
double rate = result / 10000.0; |
||||||
|
|
||||||
|
fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; \n", |
||||||
|
rate*100.0, length); |
||||||
|
|
||||||
|
//ASSERT_LE(rate, 0.02); // Must not be over 2%
|
||||||
|
if (rate > 0.0125) |
||||||
|
mediocre_filters++; // Allowed, but not too often
|
||||||
|
else |
||||||
|
good_filters++; |
||||||
|
} |
||||||
|
|
||||||
|
fprintf(stderr, "Filters: %d good, %d mediocre\n", |
||||||
|
good_filters, mediocre_filters); |
||||||
|
|
||||||
|
ASSERT_LE(mediocre_filters, good_filters/5); |
||||||
|
} |
||||||
|
|
||||||
|
// Different bits-per-byte
|
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
google::ParseCommandLineFlags(&argc, &argv, true); |
||||||
|
|
||||||
|
return rocksdb::test::RunAllTests(); |
||||||
|
} |
||||||
@ -0,0 +1,470 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "util/hash_linklist_rep.h" |
||||||
|
|
||||||
|
#include "rocksdb/memtablerep.h" |
||||||
|
#include "util/arena.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/slice_transform.h" |
||||||
|
#include "port/port.h" |
||||||
|
#include "port/atomic_pointer.h" |
||||||
|
#include "util/murmurhash.h" |
||||||
|
#include "db/memtable.h" |
||||||
|
#include "db/skiplist.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
namespace { |
||||||
|
|
||||||
|
typedef const char* Key; |
||||||
|
|
||||||
|
struct Node { |
||||||
|
explicit Node(const Key& k) : |
||||||
|
key(k) { |
||||||
|
} |
||||||
|
|
||||||
|
Key const key; |
||||||
|
|
||||||
|
// Accessors/mutators for links. Wrapped in methods so we can
|
||||||
|
// add the appropriate barriers as necessary.
|
||||||
|
Node* Next() { |
||||||
|
// Use an 'acquire load' so that we observe a fully initialized
|
||||||
|
// version of the returned Node.
|
||||||
|
return reinterpret_cast<Node*>(next_.Acquire_Load()); |
||||||
|
} |
||||||
|
void SetNext(Node* x) { |
||||||
|
// Use a 'release store' so that anybody who reads through this
|
||||||
|
// pointer observes a fully initialized version of the inserted node.
|
||||||
|
next_.Release_Store(x); |
||||||
|
} |
||||||
|
|
||||||
|
// No-barrier variants that can be safely used in a few locations.
|
||||||
|
Node* NoBarrier_Next() { |
||||||
|
return reinterpret_cast<Node*>(next_.NoBarrier_Load()); |
||||||
|
} |
||||||
|
void NoBarrier_SetNext(Node* x) { |
||||||
|
next_.NoBarrier_Store(x); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
port::AtomicPointer next_; |
||||||
|
}; |
||||||
|
|
||||||
|
class HashLinkListRep : public MemTableRep { |
||||||
|
public: |
||||||
|
HashLinkListRep(MemTableRep::KeyComparator& compare, Arena* arena, |
||||||
|
const SliceTransform* transform, size_t bucket_size); |
||||||
|
|
||||||
|
virtual void Insert(const char* key) override; |
||||||
|
|
||||||
|
virtual bool Contains(const char* key) const override; |
||||||
|
|
||||||
|
virtual size_t ApproximateMemoryUsage() override; |
||||||
|
|
||||||
|
virtual ~HashLinkListRep(); |
||||||
|
|
||||||
|
virtual MemTableRep::Iterator* GetIterator() override; |
||||||
|
|
||||||
|
virtual MemTableRep::Iterator* GetIterator(const Slice& slice) override; |
||||||
|
|
||||||
|
virtual MemTableRep::Iterator* GetPrefixIterator(const Slice& prefix) |
||||||
|
override; |
||||||
|
|
||||||
|
virtual MemTableRep::Iterator* GetDynamicPrefixIterator() override; |
||||||
|
|
||||||
|
private: |
||||||
|
friend class DynamicIterator; |
||||||
|
typedef SkipList<const char*, MemTableRep::KeyComparator&> FullList; |
||||||
|
|
||||||
|
size_t bucket_size_; |
||||||
|
|
||||||
|
// Maps slices (which are transformed user keys) to buckets of keys sharing
|
||||||
|
// the same transform.
|
||||||
|
port::AtomicPointer* buckets_; |
||||||
|
|
||||||
|
// The user-supplied transform whose domain is the user keys.
|
||||||
|
const SliceTransform* transform_; |
||||||
|
|
||||||
|
MemTableRep::KeyComparator& compare_; |
||||||
|
// immutable after construction
|
||||||
|
Arena* const arena_; |
||||||
|
|
||||||
|
bool BucketContains(Node* head, const Slice& key) const; |
||||||
|
|
||||||
|
Slice GetPrefix(const Slice& internal_key) const { |
||||||
|
return transform_->Transform(ExtractUserKey(internal_key)); |
||||||
|
} |
||||||
|
|
||||||
|
size_t GetHash(const Slice& slice) const { |
||||||
|
return MurmurHash(slice.data(), slice.size(), 0) % bucket_size_; |
||||||
|
} |
||||||
|
|
||||||
|
Node* GetBucket(size_t i) const { |
||||||
|
return static_cast<Node*>(buckets_[i].Acquire_Load()); |
||||||
|
} |
||||||
|
|
||||||
|
Node* GetBucket(const Slice& slice) const { |
||||||
|
return GetBucket(GetHash(slice)); |
||||||
|
} |
||||||
|
|
||||||
|
Node* NewNode(const Key& key) { |
||||||
|
char* mem = arena_->AllocateAligned(sizeof(Node)); |
||||||
|
return new (mem) Node(key); |
||||||
|
} |
||||||
|
|
||||||
|
bool Equal(const Slice& a, const Key& b) const { |
||||||
|
return (compare_(b, a) == 0); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } |
||||||
|
|
||||||
|
bool KeyIsAfterNode(const Slice& internal_key, const Node* n) const { |
||||||
|
// nullptr n is considered infinite
|
||||||
|
return (n != nullptr) && (compare_(n->key, internal_key) < 0); |
||||||
|
} |
||||||
|
|
||||||
|
bool KeyIsAfterNode(const Key& key, const Node* n) const { |
||||||
|
// nullptr n is considered infinite
|
||||||
|
return (n != nullptr) && (compare_(n->key, key) < 0); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
Node* FindGreaterOrEqualInBucket(Node* head, const Slice& key) const; |
||||||
|
|
||||||
|
class FullListIterator : public MemTableRep::Iterator { |
||||||
|
public: |
||||||
|
explicit FullListIterator(FullList* list) |
||||||
|
: iter_(list), full_list_(list) {} |
||||||
|
|
||||||
|
virtual ~FullListIterator() { |
||||||
|
} |
||||||
|
|
||||||
|
// Returns true iff the iterator is positioned at a valid node.
|
||||||
|
virtual bool Valid() const { |
||||||
|
return iter_.Valid(); |
||||||
|
} |
||||||
|
|
||||||
|
// Returns the key at the current position.
|
||||||
|
// REQUIRES: Valid()
|
||||||
|
virtual const char* key() const { |
||||||
|
assert(Valid()); |
||||||
|
return iter_.key(); |
||||||
|
} |
||||||
|
|
||||||
|
// Advances to the next position.
|
||||||
|
// REQUIRES: Valid()
|
||||||
|
virtual void Next() { |
||||||
|
assert(Valid()); |
||||||
|
iter_.Next(); |
||||||
|
} |
||||||
|
|
||||||
|
// Advances to the previous position.
|
||||||
|
// REQUIRES: Valid()
|
||||||
|
virtual void Prev() { |
||||||
|
assert(Valid()); |
||||||
|
iter_.Prev(); |
||||||
|
} |
||||||
|
|
||||||
|
// Advance to the first entry with a key >= target
|
||||||
|
virtual void Seek(const Slice& internal_key, const char* memtable_key) { |
||||||
|
const char* encoded_key = |
||||||
|
(memtable_key != nullptr) ? |
||||||
|
memtable_key : EncodeKey(&tmp_, internal_key); |
||||||
|
iter_.Seek(encoded_key); |
||||||
|
} |
||||||
|
|
||||||
|
// Position at the first entry in collection.
|
||||||
|
// Final state of iterator is Valid() iff collection is not empty.
|
||||||
|
virtual void SeekToFirst() { |
||||||
|
iter_.SeekToFirst(); |
||||||
|
} |
||||||
|
|
||||||
|
// Position at the last entry in collection.
|
||||||
|
// Final state of iterator is Valid() iff collection is not empty.
|
||||||
|
virtual void SeekToLast() { |
||||||
|
iter_.SeekToLast(); |
||||||
|
} |
||||||
|
private: |
||||||
|
FullList::Iterator iter_; |
||||||
|
// To destruct with the iterator.
|
||||||
|
std::unique_ptr<FullList> full_list_; |
||||||
|
std::string tmp_; // For passing to EncodeKey
|
||||||
|
}; |
||||||
|
|
||||||
|
class Iterator : public MemTableRep::Iterator { |
||||||
|
public: |
||||||
|
explicit Iterator(const HashLinkListRep* const hash_link_list_rep, |
||||||
|
Node* head) : |
||||||
|
hash_link_list_rep_(hash_link_list_rep), head_(head), node_(nullptr) { |
||||||
|
} |
||||||
|
|
||||||
|
virtual ~Iterator() { |
||||||
|
} |
||||||
|
|
||||||
|
// Returns true iff the iterator is positioned at a valid node.
|
||||||
|
virtual bool Valid() const { |
||||||
|
return node_ != nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
// Returns the key at the current position.
|
||||||
|
// REQUIRES: Valid()
|
||||||
|
virtual const char* key() const { |
||||||
|
assert(Valid()); |
||||||
|
return node_->key; |
||||||
|
} |
||||||
|
|
||||||
|
// Advances to the next position.
|
||||||
|
// REQUIRES: Valid()
|
||||||
|
virtual void Next() { |
||||||
|
assert(Valid()); |
||||||
|
node_ = node_->Next(); |
||||||
|
} |
||||||
|
|
||||||
|
// Advances to the previous position.
|
||||||
|
// REQUIRES: Valid()
|
||||||
|
virtual void Prev() { |
||||||
|
// Prefix iterator does not support total order.
|
||||||
|
// We simply set the iterator to invalid state
|
||||||
|
Reset(nullptr); |
||||||
|
} |
||||||
|
|
||||||
|
// Advance to the first entry with a key >= target
|
||||||
|
virtual void Seek(const Slice& internal_key, const char* memtable_key) { |
||||||
|
node_ = hash_link_list_rep_->FindGreaterOrEqualInBucket(head_, |
||||||
|
internal_key); |
||||||
|
} |
||||||
|
|
||||||
|
// Position at the first entry in collection.
|
||||||
|
// Final state of iterator is Valid() iff collection is not empty.
|
||||||
|
virtual void SeekToFirst() { |
||||||
|
// Prefix iterator does not support total order.
|
||||||
|
// We simply set the iterator to invalid state
|
||||||
|
Reset(nullptr); |
||||||
|
} |
||||||
|
|
||||||
|
// Position at the last entry in collection.
|
||||||
|
// Final state of iterator is Valid() iff collection is not empty.
|
||||||
|
virtual void SeekToLast() { |
||||||
|
// Prefix iterator does not support total order.
|
||||||
|
// We simply set the iterator to invalid state
|
||||||
|
Reset(nullptr); |
||||||
|
} |
||||||
|
|
||||||
|
protected: |
||||||
|
void Reset(Node* head) { |
||||||
|
head_ = head; |
||||||
|
node_ = nullptr; |
||||||
|
} |
||||||
|
private: |
||||||
|
friend class HashLinkListRep; |
||||||
|
const HashLinkListRep* const hash_link_list_rep_; |
||||||
|
Node* head_; |
||||||
|
Node* node_; |
||||||
|
std::string tmp_; // For passing to EncodeKey
|
||||||
|
|
||||||
|
virtual void SeekToHead() { |
||||||
|
node_ = head_; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
class DynamicIterator : public HashLinkListRep::Iterator { |
||||||
|
public: |
||||||
|
explicit DynamicIterator(HashLinkListRep& memtable_rep) |
||||||
|
: HashLinkListRep::Iterator(&memtable_rep, nullptr), |
||||||
|
memtable_rep_(memtable_rep) {} |
||||||
|
|
||||||
|
// Advance to the first entry with a key >= target
|
||||||
|
virtual void Seek(const Slice& k, const char* memtable_key) { |
||||||
|
auto transformed = memtable_rep_.GetPrefix(k); |
||||||
|
Reset(memtable_rep_.GetBucket(transformed)); |
||||||
|
HashLinkListRep::Iterator::Seek(k, memtable_key); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
// the underlying memtable
|
||||||
|
const HashLinkListRep& memtable_rep_; |
||||||
|
}; |
||||||
|
|
||||||
|
class EmptyIterator : public MemTableRep::Iterator { |
||||||
|
// This is used when there wasn't a bucket. It is cheaper than
|
||||||
|
// instantiating an empty bucket over which to iterate.
|
||||||
|
public: |
||||||
|
EmptyIterator() { } |
||||||
|
virtual bool Valid() const { |
||||||
|
return false; |
||||||
|
} |
||||||
|
virtual const char* key() const { |
||||||
|
assert(false); |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
virtual void Next() { } |
||||||
|
virtual void Prev() { } |
||||||
|
virtual void Seek(const Slice& user_key, const char* memtable_key) { } |
||||||
|
virtual void SeekToFirst() { } |
||||||
|
virtual void SeekToLast() { } |
||||||
|
private: |
||||||
|
}; |
||||||
|
}; |
||||||
|
|
||||||
|
HashLinkListRep::HashLinkListRep(MemTableRep::KeyComparator& compare, |
||||||
|
Arena* arena, const SliceTransform* transform, |
||||||
|
size_t bucket_size) |
||||||
|
: bucket_size_(bucket_size), |
||||||
|
transform_(transform), |
||||||
|
compare_(compare), |
||||||
|
arena_(arena) { |
||||||
|
char* mem = arena_->AllocateAligned( |
||||||
|
sizeof(port::AtomicPointer) * bucket_size); |
||||||
|
|
||||||
|
buckets_ = new (mem) port::AtomicPointer[bucket_size]; |
||||||
|
|
||||||
|
for (size_t i = 0; i < bucket_size_; ++i) { |
||||||
|
buckets_[i].NoBarrier_Store(nullptr); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
HashLinkListRep::~HashLinkListRep() { |
||||||
|
} |
||||||
|
|
||||||
|
void HashLinkListRep::Insert(const char* key) { |
||||||
|
assert(!Contains(key)); |
||||||
|
Slice internal_key = GetLengthPrefixedSlice(key); |
||||||
|
auto transformed = GetPrefix(internal_key); |
||||||
|
auto& bucket = buckets_[GetHash(transformed)]; |
||||||
|
Node* head = static_cast<Node*>(bucket.Acquire_Load()); |
||||||
|
|
||||||
|
if (!head) { |
||||||
|
Node* x = NewNode(key); |
||||||
|
// NoBarrier_SetNext() suffices since we will add a barrier when
|
||||||
|
// we publish a pointer to "x" in prev[i].
|
||||||
|
x->NoBarrier_SetNext(nullptr); |
||||||
|
bucket.Release_Store(static_cast<void*>(x)); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
Node* cur = head; |
||||||
|
Node* prev = nullptr; |
||||||
|
while (true) { |
||||||
|
if (cur == nullptr) { |
||||||
|
break; |
||||||
|
} |
||||||
|
Node* next = cur->Next(); |
||||||
|
// Make sure the lists are sorted.
|
||||||
|
// If x points to head_ or next points nullptr, it is trivially satisfied.
|
||||||
|
assert((cur == head) || (next == nullptr) || |
||||||
|
KeyIsAfterNode(next->key, cur)); |
||||||
|
if (KeyIsAfterNode(internal_key, cur)) { |
||||||
|
// Keep searching in this list
|
||||||
|
prev = cur; |
||||||
|
cur = next; |
||||||
|
} else { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Our data structure does not allow duplicate insertion
|
||||||
|
assert(cur == nullptr || !Equal(key, cur->key)); |
||||||
|
|
||||||
|
Node* x = NewNode(key); |
||||||
|
|
||||||
|
// NoBarrier_SetNext() suffices since we will add a barrier when
|
||||||
|
// we publish a pointer to "x" in prev[i].
|
||||||
|
x->NoBarrier_SetNext(cur); |
||||||
|
|
||||||
|
if (prev) { |
||||||
|
prev->SetNext(x); |
||||||
|
} else { |
||||||
|
bucket.Release_Store(static_cast<void*>(x)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
bool HashLinkListRep::Contains(const char* key) const { |
||||||
|
Slice internal_key = GetLengthPrefixedSlice(key); |
||||||
|
|
||||||
|
auto transformed = GetPrefix(internal_key); |
||||||
|
auto bucket = GetBucket(transformed); |
||||||
|
if (bucket == nullptr) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
return BucketContains(bucket, internal_key); |
||||||
|
} |
||||||
|
|
||||||
|
size_t HashLinkListRep::ApproximateMemoryUsage() { |
||||||
|
// Memory is always allocated from the arena.
|
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
MemTableRep::Iterator* HashLinkListRep::GetIterator() { |
||||||
|
auto list = new FullList(compare_, arena_); |
||||||
|
for (size_t i = 0; i < bucket_size_; ++i) { |
||||||
|
auto bucket = GetBucket(i); |
||||||
|
if (bucket != nullptr) { |
||||||
|
Iterator itr(this, bucket); |
||||||
|
for (itr.SeekToHead(); itr.Valid(); itr.Next()) { |
||||||
|
list->Insert(itr.key()); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return new FullListIterator(list); |
||||||
|
} |
||||||
|
|
||||||
|
MemTableRep::Iterator* HashLinkListRep::GetPrefixIterator( |
||||||
|
const Slice& prefix) { |
||||||
|
auto bucket = GetBucket(prefix); |
||||||
|
if (bucket == nullptr) { |
||||||
|
return new EmptyIterator(); |
||||||
|
} |
||||||
|
return new Iterator(this, bucket); |
||||||
|
} |
||||||
|
|
||||||
|
MemTableRep::Iterator* HashLinkListRep::GetIterator(const Slice& slice) { |
||||||
|
return GetPrefixIterator(transform_->Transform(slice)); |
||||||
|
} |
||||||
|
|
||||||
|
MemTableRep::Iterator* HashLinkListRep::GetDynamicPrefixIterator() { |
||||||
|
return new DynamicIterator(*this); |
||||||
|
} |
||||||
|
|
||||||
|
bool HashLinkListRep::BucketContains(Node* head, const Slice& user_key) const { |
||||||
|
Node* x = FindGreaterOrEqualInBucket(head, user_key); |
||||||
|
return (x != nullptr && Equal(user_key, x->key)); |
||||||
|
} |
||||||
|
|
||||||
|
Node* HashLinkListRep::FindGreaterOrEqualInBucket(Node* head, |
||||||
|
const Slice& key) const { |
||||||
|
Node* x = head; |
||||||
|
while (true) { |
||||||
|
if (x == nullptr) { |
||||||
|
return x; |
||||||
|
} |
||||||
|
Node* next = x->Next(); |
||||||
|
// Make sure the lists are sorted.
|
||||||
|
// If x points to head_ or next points nullptr, it is trivially satisfied.
|
||||||
|
assert((x == head) || (next == nullptr) || KeyIsAfterNode(next->key, x)); |
||||||
|
if (KeyIsAfterNode(key, x)) { |
||||||
|
// Keep searching in this list
|
||||||
|
x = next; |
||||||
|
} else { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
return x; |
||||||
|
} |
||||||
|
|
||||||
|
} // anon namespace
|
||||||
|
|
||||||
|
MemTableRep* HashLinkListRepFactory::CreateMemTableRep( |
||||||
|
MemTableRep::KeyComparator& compare, Arena* arena) { |
||||||
|
return new HashLinkListRep(compare, arena, transform_, bucket_count_); |
||||||
|
} |
||||||
|
|
||||||
|
MemTableRepFactory* NewHashLinkListRepFactory( |
||||||
|
const SliceTransform* transform, size_t bucket_count) { |
||||||
|
return new HashLinkListRepFactory(transform, bucket_count); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,39 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include "rocksdb/slice_transform.h" |
||||||
|
#include "rocksdb/memtablerep.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class HashLinkListRepFactory : public MemTableRepFactory { |
||||||
|
public: |
||||||
|
explicit HashLinkListRepFactory( |
||||||
|
const SliceTransform* transform, |
||||||
|
size_t bucket_count) |
||||||
|
: transform_(transform), |
||||||
|
bucket_count_(bucket_count) { } |
||||||
|
|
||||||
|
virtual ~HashLinkListRepFactory() { delete transform_; } |
||||||
|
|
||||||
|
virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator& compare, |
||||||
|
Arena* arena) override; |
||||||
|
|
||||||
|
virtual const char* Name() const override { |
||||||
|
return "HashLinkListRepFactory"; |
||||||
|
} |
||||||
|
|
||||||
|
const SliceTransform* GetTransform() { return transform_; } |
||||||
|
|
||||||
|
private: |
||||||
|
const SliceTransform* transform_; |
||||||
|
const size_t bucket_count_; |
||||||
|
}; |
||||||
|
|
||||||
|
} |
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue