Summary: Deprecate this one option and delete code and tests that are now superfluous. Test Plan: all tests pass Reviewers: igor, yhchiang, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: msalib, leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D55317main
sdong
8 years ago
parent
0e77246ba9
commit
1d725ca51d
@ -1,159 +0,0 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "table/block_hash_index.h" |
||||
|
||||
#include <algorithm> |
||||
|
||||
#include "rocksdb/comparator.h" |
||||
#include "rocksdb/iterator.h" |
||||
#include "rocksdb/slice_transform.h" |
||||
#include "table/internal_iterator.h" |
||||
#include "util/coding.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
Status CreateBlockHashIndex(const SliceTransform* hash_key_extractor, |
||||
const Slice& prefixes, const Slice& prefix_meta, |
||||
BlockHashIndex** hash_index) { |
||||
uint64_t pos = 0; |
||||
auto meta_pos = prefix_meta; |
||||
Status s; |
||||
*hash_index = new BlockHashIndex( |
||||
hash_key_extractor, |
||||
false /* external module manages memory space for prefixes */); |
||||
|
||||
while (!meta_pos.empty()) { |
||||
uint32_t prefix_size = 0; |
||||
uint32_t entry_index = 0; |
||||
uint32_t num_blocks = 0; |
||||
if (!GetVarint32(&meta_pos, &prefix_size) || |
||||
!GetVarint32(&meta_pos, &entry_index) || |
||||
!GetVarint32(&meta_pos, &num_blocks)) { |
||||
s = Status::Corruption( |
||||
"Corrupted prefix meta block: unable to read from it."); |
||||
break; |
||||
} |
||||
Slice prefix(prefixes.data() + pos, prefix_size); |
||||
(*hash_index)->Add(prefix, entry_index, num_blocks); |
||||
|
||||
pos += prefix_size; |
||||
} |
||||
|
||||
if (s.ok() && pos != prefixes.size()) { |
||||
s = Status::Corruption("Corrupted prefix meta block"); |
||||
} |
||||
|
||||
if (!s.ok()) { |
||||
delete *hash_index; |
||||
} |
||||
|
||||
return s; |
||||
} |
||||
|
||||
BlockHashIndex* CreateBlockHashIndexOnTheFly( |
||||
InternalIterator* index_iter, InternalIterator* data_iter, |
||||
const uint32_t num_restarts, const Comparator* comparator, |
||||
const SliceTransform* hash_key_extractor) { |
||||
assert(hash_key_extractor); |
||||
auto hash_index = new BlockHashIndex( |
||||
hash_key_extractor, |
||||
true /* hash_index will copy prefix when Add() is called */); |
||||
uint32_t current_restart_index = 0; |
||||
|
||||
std::string pending_entry_prefix; |
||||
// pending_block_num == 0 also implies there is no entry inserted at all.
|
||||
uint32_t pending_block_num = 0; |
||||
uint32_t pending_entry_index = 0; |
||||
|
||||
// scan all the entries and create a hash index based on their prefixes.
|
||||
data_iter->SeekToFirst(); |
||||
for (index_iter->SeekToFirst(); |
||||
index_iter->Valid() && current_restart_index < num_restarts; |
||||
index_iter->Next()) { |
||||
Slice last_key_in_block = index_iter->key(); |
||||
assert(data_iter->Valid() && data_iter->status().ok()); |
||||
|
||||
// scan through all entries within a data block.
|
||||
while (data_iter->Valid() && |
||||
comparator->Compare(data_iter->key(), last_key_in_block) <= 0) { |
||||
auto key_prefix = hash_key_extractor->Transform(data_iter->key()); |
||||
bool is_first_entry = pending_block_num == 0; |
||||
|
||||
// Keys may share the prefix
|
||||
if (is_first_entry || pending_entry_prefix != key_prefix) { |
||||
if (!is_first_entry) { |
||||
bool succeeded = hash_index->Add( |
||||
pending_entry_prefix, pending_entry_index, pending_block_num); |
||||
if (!succeeded) { |
||||
delete hash_index; |
||||
return nullptr; |
||||
} |
||||
} |
||||
|
||||
// update the status.
|
||||
// needs a hard copy otherwise the underlying data changes all the time.
|
||||
pending_entry_prefix = key_prefix.ToString(); |
||||
pending_block_num = 1; |
||||
pending_entry_index = current_restart_index; |
||||
} else { |
||||
// entry number increments when keys share the prefix reside in
|
||||
// different data blocks.
|
||||
auto last_restart_index = pending_entry_index + pending_block_num - 1; |
||||
assert(last_restart_index <= current_restart_index); |
||||
if (last_restart_index != current_restart_index) { |
||||
++pending_block_num; |
||||
} |
||||
} |
||||
data_iter->Next(); |
||||
} |
||||
|
||||
++current_restart_index; |
||||
} |
||||
|
||||
// make sure all entries has been scaned.
|
||||
assert(!index_iter->Valid()); |
||||
assert(!data_iter->Valid()); |
||||
|
||||
if (pending_block_num > 0) { |
||||
auto succeeded = hash_index->Add(pending_entry_prefix, pending_entry_index, |
||||
pending_block_num); |
||||
if (!succeeded) { |
||||
delete hash_index; |
||||
return nullptr; |
||||
} |
||||
} |
||||
|
||||
return hash_index; |
||||
} |
||||
|
||||
bool BlockHashIndex::Add(const Slice& prefix, uint32_t restart_index, |
||||
uint32_t num_blocks) { |
||||
auto prefix_to_insert = prefix; |
||||
if (kOwnPrefixes) { |
||||
auto prefix_ptr = arena_.Allocate(prefix.size()); |
||||
// MSVC reports C4996 Function call with parameters that may be
|
||||
// unsafe when using std::copy with a output iterator - pointer
|
||||
memcpy(prefix_ptr, prefix.data(), prefix.size()); |
||||
prefix_to_insert = Slice(prefix_ptr, prefix.size()); |
||||
} |
||||
auto result = restart_indices_.insert( |
||||
{prefix_to_insert, RestartIndex(restart_index, num_blocks)}); |
||||
return result.second; |
||||
} |
||||
|
||||
const BlockHashIndex::RestartIndex* BlockHashIndex::GetRestartIndex( |
||||
const Slice& key) { |
||||
auto key_prefix = hash_key_extractor_->Transform(key); |
||||
|
||||
auto pos = restart_indices_.find(key_prefix); |
||||
if (pos == restart_indices_.end()) { |
||||
return nullptr; |
||||
} |
||||
|
||||
return &pos->second; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -1,86 +0,0 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#pragma once |
||||
|
||||
#include <string> |
||||
#include <unordered_map> |
||||
|
||||
#include "rocksdb/status.h" |
||||
#include "util/arena.h" |
||||
#include "util/murmurhash.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class Comparator; |
||||
class InternalIterator; |
||||
class Slice; |
||||
class SliceTransform; |
||||
|
||||
// Build a hash-based index to speed up the lookup for "index block".
|
||||
// BlockHashIndex accepts a key and, if found, returns its restart index within
|
||||
// that index block.
|
||||
class BlockHashIndex { |
||||
public: |
||||
// Represents a restart index in the index block's restart array.
|
||||
struct RestartIndex { |
||||
explicit RestartIndex(uint32_t _first_index, uint32_t _num_blocks = 1) |
||||
: first_index(_first_index), num_blocks(_num_blocks) {} |
||||
|
||||
// For a given prefix, what is the restart index for the first data block
|
||||
// that contains it.
|
||||
uint32_t first_index = 0; |
||||
|
||||
// How many data blocks contains this prefix?
|
||||
uint32_t num_blocks = 1; |
||||
}; |
||||
|
||||
// @params own_prefixes indicate if we should take care the memory space for
|
||||
// the `key_prefix`
|
||||
// passed by Add()
|
||||
explicit BlockHashIndex(const SliceTransform* hash_key_extractor, |
||||
bool own_prefixes) |
||||
: hash_key_extractor_(hash_key_extractor), kOwnPrefixes(own_prefixes) {} |
||||
|
||||
// Maps a key to its restart first_index.
|
||||
// Returns nullptr if the restart first_index is found
|
||||
const RestartIndex* GetRestartIndex(const Slice& key); |
||||
|
||||
bool Add(const Slice& key_prefix, uint32_t restart_index, |
||||
uint32_t num_blocks); |
||||
|
||||
size_t ApproximateMemoryUsage() const { |
||||
return arena_.ApproximateMemoryUsage(); |
||||
} |
||||
|
||||
private: |
||||
const SliceTransform* hash_key_extractor_; |
||||
std::unordered_map<Slice, RestartIndex, murmur_hash> restart_indices_; |
||||
|
||||
Arena arena_; |
||||
bool kOwnPrefixes; |
||||
}; |
||||
|
||||
// Create hash index by reading from the metadata blocks.
|
||||
// @params prefixes: a sequence of prefixes.
|
||||
// @params prefix_meta: contains the "metadata" to of the prefixes.
|
||||
Status CreateBlockHashIndex(const SliceTransform* hash_key_extractor, |
||||
const Slice& prefixes, const Slice& prefix_meta, |
||||
BlockHashIndex** hash_index); |
||||
|
||||
// Create hash index by scanning the entries in index as well as the whole
|
||||
// dataset.
|
||||
// @params index_iter: an iterator with the pointer to the first entry in a
|
||||
// block.
|
||||
// @params data_iter: an iterator that can scan all the entries reside in a
|
||||
// table.
|
||||
// @params num_restarts: used for correctness verification.
|
||||
// @params hash_key_extractor: extract the hashable part of a given key.
|
||||
// On error, nullptr will be returned.
|
||||
BlockHashIndex* CreateBlockHashIndexOnTheFly( |
||||
InternalIterator* index_iter, InternalIterator* data_iter, |
||||
const uint32_t num_restarts, const Comparator* comparator, |
||||
const SliceTransform* hash_key_extractor); |
||||
|
||||
} // namespace rocksdb
|
||||
@ -1,121 +0,0 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <map> |
||||
#include <memory> |
||||
#include <vector> |
||||
|
||||
#include "rocksdb/comparator.h" |
||||
#include "rocksdb/iterator.h" |
||||
#include "rocksdb/slice_transform.h" |
||||
#include "table/block_hash_index.h" |
||||
#include "table/internal_iterator.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
typedef std::map<std::string, std::string> Data; |
||||
|
||||
class MapIterator : public InternalIterator { |
||||
public: |
||||
explicit MapIterator(const Data& data) : data_(data), pos_(data_.end()) {} |
||||
|
||||
virtual bool Valid() const override { return pos_ != data_.end(); } |
||||
|
||||
virtual void SeekToFirst() override { pos_ = data_.begin(); } |
||||
|
||||
virtual void SeekToLast() override { |
||||
pos_ = data_.end(); |
||||
--pos_; |
||||
} |
||||
|
||||
virtual void Seek(const Slice& target) override { |
||||
pos_ = data_.find(target.ToString()); |
||||
} |
||||
|
||||
virtual void Next() override { ++pos_; } |
||||
|
||||
virtual void Prev() override { --pos_; } |
||||
|
||||
virtual Slice key() const override { return pos_->first; } |
||||
|
||||
virtual Slice value() const override { return pos_->second; } |
||||
|
||||
virtual Status status() const override { return Status::OK(); } |
||||
|
||||
private: |
||||
const Data& data_; |
||||
Data::const_iterator pos_; |
||||
}; |
||||
|
||||
class BlockTest : public testing::Test {}; |
||||
|
||||
TEST_F(BlockTest, BasicTest) { |
||||
const size_t keys_per_block = 4; |
||||
const size_t prefix_size = 2; |
||||
std::vector<std::string> keys = {/* block 1 */ |
||||
"0101", "0102", "0103", "0201", |
||||
/* block 2 */ |
||||
"0202", "0203", "0301", "0401", |
||||
/* block 3 */ |
||||
"0501", "0601", "0701", "0801", |
||||
/* block 4 */ |
||||
"0802", "0803", "0804", "0805", |
||||
/* block 5 */ |
||||
"0806", "0807", "0808", "0809", }; |
||||
|
||||
Data data_entries; |
||||
for (const auto key : keys) { |
||||
data_entries.insert({key, key}); |
||||
} |
||||
|
||||
Data index_entries; |
||||
for (size_t i = 3; i < keys.size(); i += keys_per_block) { |
||||
// simply ignore the value part
|
||||
index_entries.insert({keys[i], ""}); |
||||
} |
||||
|
||||
MapIterator data_iter(data_entries); |
||||
MapIterator index_iter(index_entries); |
||||
|
||||
auto prefix_extractor = NewFixedPrefixTransform(prefix_size); |
||||
std::unique_ptr<BlockHashIndex> block_hash_index(CreateBlockHashIndexOnTheFly( |
||||
&index_iter, &data_iter, static_cast<uint32_t>(index_entries.size()), |
||||
BytewiseComparator(), prefix_extractor)); |
||||
|
||||
std::map<std::string, BlockHashIndex::RestartIndex> expected = { |
||||
{"01xx", BlockHashIndex::RestartIndex(0, 1)}, |
||||
{"02yy", BlockHashIndex::RestartIndex(0, 2)}, |
||||
{"03zz", BlockHashIndex::RestartIndex(1, 1)}, |
||||
{"04pp", BlockHashIndex::RestartIndex(1, 1)}, |
||||
{"05ww", BlockHashIndex::RestartIndex(2, 1)}, |
||||
{"06xx", BlockHashIndex::RestartIndex(2, 1)}, |
||||
{"07pp", BlockHashIndex::RestartIndex(2, 1)}, |
||||
{"08xz", BlockHashIndex::RestartIndex(2, 3)}, }; |
||||
|
||||
const BlockHashIndex::RestartIndex* index = nullptr; |
||||
// search existed prefixes
|
||||
for (const auto& item : expected) { |
||||
index = block_hash_index->GetRestartIndex(item.first); |
||||
ASSERT_TRUE(index != nullptr); |
||||
ASSERT_EQ(item.second.first_index, index->first_index); |
||||
ASSERT_EQ(item.second.num_blocks, index->num_blocks); |
||||
} |
||||
|
||||
// search non exist prefixes
|
||||
ASSERT_TRUE(!block_hash_index->GetRestartIndex("00xx")); |
||||
ASSERT_TRUE(!block_hash_index->GetRestartIndex("10yy")); |
||||
ASSERT_TRUE(!block_hash_index->GetRestartIndex("20zz")); |
||||
|
||||
delete prefix_extractor; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
Loading…
Reference in new issue