Summary: Contains: - Implementation of TableReader based on Cuckoo Hashing - Unittests for CuckooTableReader - Performance test for TableReader Test Plan: make cuckoo_table_reader_test ./cuckoo_table_reader_test make valgrind_check make asan_check Reviewers: yhchiang, sdong, igor, ljin Reviewed By: ljin Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D20511main
Radheshyam Balasundaram
11 years ago
parent
d650612c4c
commit
62f9b071ff
@ -0,0 +1,112 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
#include "table/cuckoo_table_reader.h" |
||||||
|
|
||||||
|
#include <string> |
||||||
|
#include "table/meta_blocks.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
extern const uint64_t kCuckooTableMagicNumber; |
||||||
|
|
||||||
|
CuckooTableReader::CuckooTableReader( |
||||||
|
const Options& options, |
||||||
|
std::unique_ptr<RandomAccessFile>&& file, |
||||||
|
uint64_t file_size, |
||||||
|
uint64_t (*GetSliceHashPtr)(const Slice&, uint32_t, uint64_t)) |
||||||
|
: file_(std::move(file)), |
||||||
|
file_size_(file_size), |
||||||
|
GetSliceHash(GetSliceHashPtr) { |
||||||
|
if (!options.allow_mmap_reads) { |
||||||
|
status_ = Status::InvalidArgument("File is not mmaped"); |
||||||
|
} |
||||||
|
TableProperties* props = nullptr; |
||||||
|
status_ = ReadTableProperties(file_.get(), file_size, kCuckooTableMagicNumber, |
||||||
|
options.env, options.info_log.get(), &props); |
||||||
|
if (!status_.ok()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
table_props_.reset(props); |
||||||
|
auto& user_props = props->user_collected_properties; |
||||||
|
auto hash_funs = user_props.find(CuckooTablePropertyNames::kNumHashTable); |
||||||
|
if (hash_funs == user_props.end()) { |
||||||
|
status_ = Status::InvalidArgument("Number of hash functions not found"); |
||||||
|
return; |
||||||
|
} |
||||||
|
num_hash_fun_ = *reinterpret_cast<const uint32_t*>(hash_funs->second.data()); |
||||||
|
auto unused_key = user_props.find(CuckooTablePropertyNames::kEmptyKey); |
||||||
|
if (unused_key == user_props.end()) { |
||||||
|
status_ = Status::InvalidArgument("Empty bucket value not found"); |
||||||
|
return; |
||||||
|
} |
||||||
|
unused_key_ = unused_key->second; |
||||||
|
|
||||||
|
key_length_ = props->fixed_key_len; |
||||||
|
auto value_length = user_props.find(CuckooTablePropertyNames::kValueLength); |
||||||
|
if (value_length == user_props.end()) { |
||||||
|
status_ = Status::InvalidArgument("Value length not found"); |
||||||
|
return; |
||||||
|
} |
||||||
|
value_length_ = *reinterpret_cast<const uint32_t*>( |
||||||
|
value_length->second.data()); |
||||||
|
bucket_length_ = key_length_ + value_length_; |
||||||
|
|
||||||
|
auto num_buckets = user_props.find(CuckooTablePropertyNames::kMaxNumBuckets); |
||||||
|
if (num_buckets == user_props.end()) { |
||||||
|
status_ = Status::InvalidArgument("Num buckets not found"); |
||||||
|
return; |
||||||
|
} |
||||||
|
num_buckets_ = *reinterpret_cast<const uint64_t*>(num_buckets->second.data()); |
||||||
|
auto is_last_level = user_props.find(CuckooTablePropertyNames::kIsLastLevel); |
||||||
|
if (is_last_level == user_props.end()) { |
||||||
|
status_ = Status::InvalidArgument("Is last level not found"); |
||||||
|
return; |
||||||
|
} |
||||||
|
is_last_level_ = *reinterpret_cast<const bool*>(is_last_level->second.data()); |
||||||
|
status_ = file_->Read(0, file_size, &file_data_, nullptr); |
||||||
|
} |
||||||
|
|
||||||
|
Status CuckooTableReader::Get( |
||||||
|
const ReadOptions& readOptions, const Slice& key, void* handle_context, |
||||||
|
bool (*result_handler)(void* arg, const ParsedInternalKey& k, |
||||||
|
const Slice& v), |
||||||
|
void (*mark_key_may_exist_handler)(void* handle_context)) { |
||||||
|
ParsedInternalKey ikey; |
||||||
|
if (!ParseInternalKey(key, &ikey)) { |
||||||
|
return Status::Corruption("Unable to parse key into inernal key."); |
||||||
|
} |
||||||
|
for (uint32_t hash_cnt = 0; hash_cnt < num_hash_fun_; ++hash_cnt) { |
||||||
|
uint64_t hash_val = GetSliceHash(ikey.user_key, hash_cnt, num_buckets_); |
||||||
|
assert(hash_val < num_buckets_); |
||||||
|
uint64_t offset = hash_val * bucket_length_; |
||||||
|
const char* bucket = &file_data_.data()[offset]; |
||||||
|
if (unused_key_.compare(0, key_length_, bucket, key_length_) == 0) { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
// Here, we compare only the user key part as we support only one entry
|
||||||
|
// per user key and we don't support sanpshot.
|
||||||
|
if (ikey.user_key.compare(Slice(bucket, ikey.user_key.size())) == 0) { |
||||||
|
Slice value = Slice(&bucket[key_length_], value_length_); |
||||||
|
result_handler(handle_context, ikey, value); |
||||||
|
// We don't support merge operations. So, we return here.
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Iterator* CuckooTableReader::NewIterator(const ReadOptions&, Arena* arena) { |
||||||
|
// TODO(rbs): Implement this as this will be used in compaction.
|
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
} // namespace rocksdb
|
||||||
|
#endif |
||||||
@ -0,0 +1,71 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
#include <string> |
||||||
|
#include <memory> |
||||||
|
|
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "table/table_reader.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Arena; |
||||||
|
class TableReader; |
||||||
|
|
||||||
|
class CuckooTableReader: public TableReader { |
||||||
|
public: |
||||||
|
CuckooTableReader( |
||||||
|
const Options& options, |
||||||
|
std::unique_ptr<RandomAccessFile>&& file, |
||||||
|
uint64_t file_size, |
||||||
|
uint64_t (*GetSliceHash)(const Slice&, uint32_t, uint64_t)); |
||||||
|
~CuckooTableReader() {} |
||||||
|
|
||||||
|
std::shared_ptr<const TableProperties> GetTableProperties() const { |
||||||
|
return table_props_; |
||||||
|
} |
||||||
|
|
||||||
|
Status status() const { return status_; } |
||||||
|
|
||||||
|
Status Get( |
||||||
|
const ReadOptions& readOptions, const Slice& key, void* handle_context, |
||||||
|
bool (*result_handler)(void* arg, const ParsedInternalKey& k, |
||||||
|
const Slice& v), |
||||||
|
void (*mark_key_may_exist_handler)(void* handle_context) = nullptr); |
||||||
|
|
||||||
|
Iterator* NewIterator(const ReadOptions&, Arena* arena = nullptr); |
||||||
|
|
||||||
|
// Following methods are not implemented for Cuckoo Table Reader
|
||||||
|
uint64_t ApproximateOffsetOf(const Slice& key) { return 0; } |
||||||
|
void SetupForCompaction() {} |
||||||
|
void Prepare(const Slice& target) {} |
||||||
|
// End of methods not implemented.
|
||||||
|
|
||||||
|
private: |
||||||
|
std::unique_ptr<RandomAccessFile> file_; |
||||||
|
Slice file_data_; |
||||||
|
const uint64_t file_size_; |
||||||
|
bool is_last_level_; |
||||||
|
std::shared_ptr<const TableProperties> table_props_; |
||||||
|
Status status_; |
||||||
|
uint32_t num_hash_fun_; |
||||||
|
std::string unused_key_; |
||||||
|
uint32_t key_length_; |
||||||
|
uint32_t value_length_; |
||||||
|
uint32_t bucket_length_; |
||||||
|
uint64_t num_buckets_; |
||||||
|
uint64_t (*GetSliceHash)(const Slice& s, uint32_t index, |
||||||
|
uint64_t max_num_buckets); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,353 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#ifndef GFLAGS |
||||||
|
#include <cstdio> |
||||||
|
int main() { |
||||||
|
fprintf(stderr, "Please install gflags to run this test\n"); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
#else |
||||||
|
|
||||||
|
#include <gflags/gflags.h> |
||||||
|
#include <vector> |
||||||
|
#include <string> |
||||||
|
#include <map> |
||||||
|
|
||||||
|
#include "table/meta_blocks.h" |
||||||
|
#include "table/cuckoo_table_builder.h" |
||||||
|
#include "table/cuckoo_table_reader.h" |
||||||
|
#include "table/cuckoo_table_factory.h" |
||||||
|
#include "util/random.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
|
||||||
|
using GFLAGS::ParseCommandLineFlags; |
||||||
|
using GFLAGS::SetUsageMessage; |
||||||
|
|
||||||
|
DEFINE_string(file_dir, "", "Directory where the files will be created" |
||||||
|
" for benchmark. Added for using tmpfs."); |
||||||
|
DEFINE_bool(enable_perf, false, "Run Benchmark Tests too."); |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
extern const uint64_t kCuckooTableMagicNumber; |
||||||
|
|
||||||
|
namespace { |
||||||
|
const uint32_t kNumHashFunc = 10; |
||||||
|
// Methods, variables related to Hash functions.
|
||||||
|
std::unordered_map<std::string, std::vector<uint64_t>> hash_map; |
||||||
|
|
||||||
|
void AddHashLookups(const std::string& s, uint64_t bucket_id, |
||||||
|
uint32_t num_hash_fun) { |
||||||
|
std::vector<uint64_t> v; |
||||||
|
for (uint32_t i = 0; i < num_hash_fun; i++) { |
||||||
|
v.push_back(bucket_id + i); |
||||||
|
} |
||||||
|
hash_map[s] = v; |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t GetSliceHash(const Slice& s, uint32_t index, |
||||||
|
uint64_t max_num_buckets) { |
||||||
|
return hash_map[s.ToString()][index]; |
||||||
|
} |
||||||
|
|
||||||
|
// Methods, variables for checking key and values read.
|
||||||
|
struct ValuesToAssert { |
||||||
|
ValuesToAssert(const std::string& key, const Slice& value) |
||||||
|
: expected_user_key(key), |
||||||
|
expected_value(value), |
||||||
|
call_count(0) {} |
||||||
|
std::string expected_user_key; |
||||||
|
Slice expected_value; |
||||||
|
int call_count; |
||||||
|
}; |
||||||
|
|
||||||
|
bool AssertValues(void* assert_obj, |
||||||
|
const ParsedInternalKey& k, const Slice& v) { |
||||||
|
ValuesToAssert *ptr = reinterpret_cast<ValuesToAssert*>(assert_obj); |
||||||
|
ASSERT_EQ(ptr->expected_value.ToString(), v.ToString()); |
||||||
|
ASSERT_EQ(ptr->expected_user_key, k.user_key.ToString()); |
||||||
|
++ptr->call_count; |
||||||
|
return false; |
||||||
|
} |
||||||
|
} // namespace
|
||||||
|
|
||||||
|
class CuckooReaderTest { |
||||||
|
public: |
||||||
|
CuckooReaderTest() { |
||||||
|
options.allow_mmap_reads = true; |
||||||
|
env = options.env; |
||||||
|
env_options = EnvOptions(options); |
||||||
|
} |
||||||
|
|
||||||
|
void SetUp(int num_items) { |
||||||
|
this->num_items = num_items; |
||||||
|
hash_map.clear(); |
||||||
|
keys.clear(); |
||||||
|
keys.resize(num_items); |
||||||
|
user_keys.clear(); |
||||||
|
user_keys.resize(num_items); |
||||||
|
values.clear(); |
||||||
|
values.resize(num_items); |
||||||
|
} |
||||||
|
|
||||||
|
void CreateCuckooFile(bool is_last_level) { |
||||||
|
unique_ptr<WritableFile> writable_file; |
||||||
|
ASSERT_OK(env->NewWritableFile(fname, &writable_file, env_options)); |
||||||
|
CuckooTableBuilder builder( |
||||||
|
writable_file.get(), keys[0].size(), values[0].size(), 0.9, |
||||||
|
10000, kNumHashFunc, 100, is_last_level, GetSliceHash); |
||||||
|
ASSERT_OK(builder.status()); |
||||||
|
for (uint32_t key_idx = 0; key_idx < num_items; ++key_idx) { |
||||||
|
builder.Add(Slice(keys[key_idx]), Slice(values[key_idx])); |
||||||
|
ASSERT_EQ(builder.NumEntries(), key_idx + 1); |
||||||
|
ASSERT_OK(builder.status()); |
||||||
|
} |
||||||
|
ASSERT_OK(builder.Finish()); |
||||||
|
ASSERT_EQ(num_items, builder.NumEntries()); |
||||||
|
file_size = builder.FileSize(); |
||||||
|
ASSERT_OK(writable_file->Close()); |
||||||
|
} |
||||||
|
|
||||||
|
void CheckReader() { |
||||||
|
unique_ptr<RandomAccessFile> read_file; |
||||||
|
ASSERT_OK(env->NewRandomAccessFile(fname, &read_file, env_options)); |
||||||
|
CuckooTableReader reader( |
||||||
|
options, |
||||||
|
std::move(read_file), |
||||||
|
file_size, |
||||||
|
GetSliceHash); |
||||||
|
ASSERT_OK(reader.status()); |
||||||
|
|
||||||
|
for (uint32_t i = 0; i < num_items; ++i) { |
||||||
|
ValuesToAssert v(user_keys[i], values[i]); |
||||||
|
ASSERT_OK(reader.Get( |
||||||
|
ReadOptions(), Slice(keys[i]), &v, AssertValues, nullptr)); |
||||||
|
ASSERT_EQ(1, v.call_count); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
std::vector<std::string> keys; |
||||||
|
std::vector<std::string> user_keys; |
||||||
|
std::vector<std::string> values; |
||||||
|
uint32_t num_items; |
||||||
|
std::string fname; |
||||||
|
uint64_t file_size; |
||||||
|
Options options; |
||||||
|
Env* env; |
||||||
|
EnvOptions env_options; |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(CuckooReaderTest, WhenKeyExists) { |
||||||
|
SetUp(10); |
||||||
|
fname = test::TmpDir() + "/CuckooReader_WhenKeyExists"; |
||||||
|
for (uint32_t i = 0; i < num_items; i++) { |
||||||
|
user_keys[i] = "keys" + std::to_string(i+100); |
||||||
|
ParsedInternalKey ikey(user_keys[i], i + 1000, kTypeValue); |
||||||
|
AppendInternalKey(&keys[i], ikey); |
||||||
|
values[i] = "value" + std::to_string(i+100); |
||||||
|
AddHashLookups(user_keys[i], i * kNumHashFunc, kNumHashFunc); |
||||||
|
} |
||||||
|
CreateCuckooFile(false); |
||||||
|
CheckReader(); |
||||||
|
// Last level file.
|
||||||
|
CreateCuckooFile(true); |
||||||
|
CheckReader(); |
||||||
|
// Test with collision. Make all hash values collide.
|
||||||
|
hash_map.clear(); |
||||||
|
for (uint32_t i = 0; i < num_items; i++) { |
||||||
|
AddHashLookups(user_keys[i], 0, kNumHashFunc); |
||||||
|
} |
||||||
|
CreateCuckooFile(false); |
||||||
|
CheckReader(); |
||||||
|
// Last level file.
|
||||||
|
CreateCuckooFile(true); |
||||||
|
CheckReader(); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(CuckooReaderTest, WhenKeyNotFound) { |
||||||
|
// Add keys with colliding hash values.
|
||||||
|
SetUp(kNumHashFunc / 2); |
||||||
|
fname = test::TmpDir() + "/CuckooReader_WhenKeyNotFound"; |
||||||
|
for (uint32_t i = 0; i < num_items; i++) { |
||||||
|
user_keys[i] = "keys" + std::to_string(i+100); |
||||||
|
ParsedInternalKey ikey(user_keys[i], i + 1000, kTypeValue); |
||||||
|
AppendInternalKey(&keys[i], ikey); |
||||||
|
values[i] = "value" + std::to_string(i+100); |
||||||
|
// Make all hash values collide.
|
||||||
|
AddHashLookups(user_keys[i], 0, kNumHashFunc); |
||||||
|
} |
||||||
|
CreateCuckooFile(false); |
||||||
|
CheckReader(); |
||||||
|
unique_ptr<RandomAccessFile> read_file; |
||||||
|
ASSERT_OK(env->NewRandomAccessFile(fname, &read_file, env_options)); |
||||||
|
CuckooTableReader reader( |
||||||
|
options, |
||||||
|
std::move(read_file), |
||||||
|
file_size, |
||||||
|
GetSliceHash); |
||||||
|
ASSERT_OK(reader.status()); |
||||||
|
// Search for a key with colliding hash values.
|
||||||
|
std::string not_found_user_key = "keys" + std::to_string(num_items + 100); |
||||||
|
std::string not_found_key; |
||||||
|
AddHashLookups(not_found_user_key, 0, kNumHashFunc); |
||||||
|
ParsedInternalKey ikey(not_found_user_key, 1000, kTypeValue); |
||||||
|
AppendInternalKey(¬_found_key, ikey); |
||||||
|
ValuesToAssert v("", ""); |
||||||
|
ASSERT_OK(reader.Get( |
||||||
|
ReadOptions(), Slice(not_found_key), &v, AssertValues, nullptr)); |
||||||
|
ASSERT_EQ(0, v.call_count); |
||||||
|
ASSERT_OK(reader.status()); |
||||||
|
// Search for a key with an independent hash value.
|
||||||
|
std::string not_found_user_key2 = "keys" + std::to_string(num_items + 101); |
||||||
|
std::string not_found_key2; |
||||||
|
AddHashLookups(not_found_user_key2, kNumHashFunc, kNumHashFunc); |
||||||
|
ParsedInternalKey ikey2(not_found_user_key2, 1000, kTypeValue); |
||||||
|
AppendInternalKey(¬_found_key2, ikey2); |
||||||
|
ASSERT_OK(reader.Get( |
||||||
|
ReadOptions(), Slice(not_found_key2), &v, AssertValues, nullptr)); |
||||||
|
ASSERT_EQ(0, v.call_count); |
||||||
|
ASSERT_OK(reader.status()); |
||||||
|
|
||||||
|
// Test read with corrupted key.
|
||||||
|
not_found_key2.pop_back(); |
||||||
|
ASSERT_TRUE(!ParseInternalKey(not_found_key2, &ikey)); |
||||||
|
ASSERT_TRUE(reader.Get( |
||||||
|
ReadOptions(), Slice(not_found_key2), &v, |
||||||
|
AssertValues, nullptr).IsCorruption()); |
||||||
|
ASSERT_EQ(0, v.call_count); |
||||||
|
ASSERT_OK(reader.status()); |
||||||
|
|
||||||
|
// Test read when key is unused key.
|
||||||
|
std::string unused_user_key = "keys10:"; |
||||||
|
// Add hash values that map to empty buckets.
|
||||||
|
AddHashLookups(unused_user_key, kNumHashFunc, kNumHashFunc); |
||||||
|
std::string unused_key; |
||||||
|
ParsedInternalKey ikey3(unused_user_key, 1000, kTypeValue); |
||||||
|
AppendInternalKey(&unused_key, ikey3); |
||||||
|
ASSERT_OK(reader.Get( |
||||||
|
ReadOptions(), Slice(unused_key), &v, AssertValues, nullptr)); |
||||||
|
ASSERT_EQ(0, v.call_count); |
||||||
|
ASSERT_OK(reader.status()); |
||||||
|
} |
||||||
|
|
||||||
|
// Performance tests
|
||||||
|
namespace { |
||||||
|
void GenerateKeys(uint64_t num, std::vector<std::string>* keys, |
||||||
|
uint32_t user_key_length) { |
||||||
|
for (uint64_t i = 0; i < num; ++i) { |
||||||
|
std::string new_key(reinterpret_cast<char*>(&i), sizeof(i)); |
||||||
|
new_key = std::string(user_key_length - new_key.size(), 'k') + new_key; |
||||||
|
ParsedInternalKey ikey(new_key, num, kTypeValue); |
||||||
|
std::string full_key; |
||||||
|
AppendInternalKey(&full_key, ikey); |
||||||
|
keys->push_back(full_key); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
bool DoNothing(void* arg, const ParsedInternalKey& k, const Slice& v) { |
||||||
|
// Deliberately empty.
|
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
bool CheckValue(void* cnt_ptr, const ParsedInternalKey& k, const Slice& v) { |
||||||
|
++*reinterpret_cast<int*>(cnt_ptr); |
||||||
|
std::string expected_value; |
||||||
|
AppendInternalKey(&expected_value, k); |
||||||
|
ASSERT_EQ(0, v.compare(Slice(&expected_value[0], v.size()))); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
// Create last level file as we are interested in measuring performance of
|
||||||
|
// last level file only.
|
||||||
|
void BM_CuckooRead(uint64_t num, uint32_t key_length, |
||||||
|
uint32_t value_length, uint64_t num_reads, double hash_ratio) { |
||||||
|
assert(value_length <= key_length); |
||||||
|
std::vector<std::string> keys; |
||||||
|
Options options; |
||||||
|
options.allow_mmap_reads = true; |
||||||
|
Env* env = options.env; |
||||||
|
EnvOptions env_options = EnvOptions(options); |
||||||
|
uint64_t file_size; |
||||||
|
if (FLAGS_file_dir.empty()) { |
||||||
|
FLAGS_file_dir = test::TmpDir(); |
||||||
|
} |
||||||
|
std::string fname = FLAGS_file_dir + "/cuckoo_read_benchmark"; |
||||||
|
|
||||||
|
GenerateKeys(num, &keys, key_length); |
||||||
|
uint64_t predicted_file_size = |
||||||
|
num * (key_length + value_length) / hash_ratio + 1024; |
||||||
|
|
||||||
|
unique_ptr<WritableFile> writable_file; |
||||||
|
ASSERT_OK(env->NewWritableFile(fname, &writable_file, env_options)); |
||||||
|
CuckooTableBuilder builder( |
||||||
|
writable_file.get(), keys[0].size(), value_length, hash_ratio, |
||||||
|
predicted_file_size, kMaxNumHashTable, 1000, true, GetSliceMurmurHash); |
||||||
|
ASSERT_OK(builder.status()); |
||||||
|
for (uint32_t key_idx = 0; key_idx < num; ++key_idx) { |
||||||
|
// Value is just a part of key.
|
||||||
|
builder.Add(Slice(keys[key_idx]), Slice(&keys[key_idx][0], value_length)); |
||||||
|
ASSERT_EQ(builder.NumEntries(), key_idx + 1); |
||||||
|
ASSERT_OK(builder.status()); |
||||||
|
} |
||||||
|
ASSERT_OK(builder.Finish()); |
||||||
|
ASSERT_EQ(num, builder.NumEntries()); |
||||||
|
file_size = builder.FileSize(); |
||||||
|
ASSERT_OK(writable_file->Close()); |
||||||
|
unique_ptr<RandomAccessFile> read_file; |
||||||
|
ASSERT_OK(env->NewRandomAccessFile(fname, &read_file, env_options)); |
||||||
|
|
||||||
|
CuckooTableReader reader( |
||||||
|
options, |
||||||
|
std::move(read_file), |
||||||
|
file_size, |
||||||
|
GetSliceMurmurHash); |
||||||
|
ASSERT_OK(reader.status()); |
||||||
|
const UserCollectedProperties user_props = |
||||||
|
reader.GetTableProperties()->user_collected_properties; |
||||||
|
const uint32_t num_hash_fun = *reinterpret_cast<const uint32_t*>( |
||||||
|
user_props.at(CuckooTablePropertyNames::kNumHashTable).data()); |
||||||
|
fprintf(stderr, "With %lu items and hash table ratio %f, number of hash" |
||||||
|
" functions used: %u.\n", num, hash_ratio, num_hash_fun); |
||||||
|
ReadOptions r_options; |
||||||
|
for (auto& key : keys) { |
||||||
|
int cnt = 0; |
||||||
|
ASSERT_OK(reader.Get(r_options, Slice(key), &cnt, CheckValue, nullptr)); |
||||||
|
ASSERT_EQ(1, cnt); |
||||||
|
} |
||||||
|
// Shuffle Keys.
|
||||||
|
std::random_shuffle(keys.begin(), keys.end()); |
||||||
|
|
||||||
|
uint64_t time_now = env->NowMicros(); |
||||||
|
for (uint64_t i = 0; i < num_reads; ++i) { |
||||||
|
reader.Get(r_options, Slice(keys[i % num]), nullptr, DoNothing, nullptr); |
||||||
|
} |
||||||
|
fprintf(stderr, "Time taken per op is %.3fus\n", |
||||||
|
(env->NowMicros() - time_now)*1.0/num_reads); |
||||||
|
} |
||||||
|
} // namespace.
|
||||||
|
|
||||||
|
TEST(CuckooReaderTest, Performance) { |
||||||
|
// In all these tests, num_reads = 10*num_items.
|
||||||
|
if (!FLAGS_enable_perf) { |
||||||
|
return; |
||||||
|
} |
||||||
|
BM_CuckooRead(100000, 8, 4, 1000000, 0.9); |
||||||
|
BM_CuckooRead(1000000, 8, 4, 10000000, 0.9); |
||||||
|
BM_CuckooRead(1000000, 8, 4, 10000000, 0.7); |
||||||
|
BM_CuckooRead(10000000, 8, 4, 100000000, 0.9); |
||||||
|
BM_CuckooRead(10000000, 8, 4, 100000000, 0.7); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
ParseCommandLineFlags(&argc, &argv, true); |
||||||
|
rocksdb::test::RunAllTests(); |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
#endif // GFLAGS.
|
||||||
Loading…
Reference in new issue