Introduce a new MultiGet batching implementation (#5011)
Summary: This PR introduces a new MultiGet() API, with the underlying implementation grouping keys based on SST file and batching lookups in a file. The reason for the new API is twofold - the definition allows callers to allocate storage for status and values on stack instead of std::vector, as well as return values as PinnableSlices in order to avoid copying, and it keeps the original MultiGet() implementation intact while we experiment with batching. Batching is useful when there is some spatial locality to the keys being queries, as well as larger batch sizes. The main benefits are due to - 1. Fewer function calls, especially to BlockBasedTableReader::MultiGet() and FullFilterBlockReader::KeysMayMatch() 2. Bloom filter cachelines can be prefetched, hiding the cache miss latency The next step is to optimize the binary searches in the level_storage_info, index blocks and data blocks, since we could reduce the number of key comparisons if the keys are relatively close to each other. The batching optimizations also need to be extended to other formats, such as PlainTable and filter formats. This also needs to be added to db_stress. Benchmark results from db_bench for various batch size/locality of reference combinations are given below. Locality was simulated by offsetting the keys in a batch by a stride length. Each SST file is about 8.6MB uncompressed and key/value size is 16/100 uncompressed. To focus on the cpu benefit of batching, the runs were single threaded and bound to the same cpu to eliminate interference from other system events. The results show a 10-25% improvement in micros/op from smaller to larger batch sizes (4 - 32). Batch Sizes 1 | 2 | 4 | 8 | 16 | 32 Random pattern (Stride length 0) 4.158 | 4.109 | 4.026 | 4.05 | 4.1 | 4.074 - Get 4.438 | 4.302 | 4.165 | 4.122 | 4.096 | 4.075 - MultiGet (no batching) 4.461 | 4.256 | 4.277 | 4.11 | 4.182 | 4.14 - MultiGet (w/ batching) Good locality (Stride length 16) 4.048 | 3.659 | 3.248 | 2.99 | 2.84 | 2.753 4.429 | 3.728 | 3.406 | 3.053 | 2.911 | 2.781 4.452 | 3.45 | 2.833 | 2.451 | 2.233 | 2.135 Good locality (Stride length 256) 4.066 | 3.786 | 3.581 | 3.447 | 3.415 | 3.232 4.406 | 4.005 | 3.644 | 3.49 | 3.381 | 3.268 4.393 | 3.649 | 3.186 | 2.882 | 2.676 | 2.62 Medium locality (Stride length 4096) 4.012 | 3.922 | 3.768 | 3.61 | 3.582 | 3.555 4.364 | 4.057 | 3.791 | 3.65 | 3.57 | 3.465 4.479 | 3.758 | 3.316 | 3.077 | 2.959 | 2.891 dbbench command used (on a DB with 4 levels, 12 million keys)- TEST_TMPDIR=/dev/shm numactl -C 10 ./db_bench.tmp -use_existing_db=true -benchmarks="readseq,multireadrandom" -write_buffer_size=4194304 -target_file_size_base=4194304 -max_bytes_for_level_base=16777216 -num=12000000 -reads=12000000 -duration=90 -threads=1 -compression_type=none -cache_size=4194304000 -batch_size=32 -disable_auto_compactions=true -bloom_bits=10 -cache_index_and_filter_blocks=true -pin_l0_filter_and_index_blocks_in_cache=true -multiread_batched=true -multiread_stride=4 Pull Request resolved: https://github.com/facebook/rocksdb/pull/5011 Differential Revision: D14348703 Pulled By: anand1976 fbshipit-source-id: 774406dab3776d979c809522a67bedac6c17f84bmain
anand76
5 years ago
committed by
Facebook Github Bot
parent
ed9f5e21aa
commit
fefd4b98c5
@ -0,0 +1,65 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once |
||||
#include <string> |
||||
#include <utility> |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/slice.h" |
||||
#include "rocksdb/types.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// A helper class useful for DBImpl::Get()
|
||||
class LookupKey { |
||||
public: |
||||
// Initialize *this for looking up user_key at a snapshot with
|
||||
// the specified sequence number.
|
||||
LookupKey(const Slice& _user_key, SequenceNumber sequence); |
||||
|
||||
~LookupKey(); |
||||
|
||||
// Return a key suitable for lookup in a MemTable.
|
||||
Slice memtable_key() const { |
||||
return Slice(start_, static_cast<size_t>(end_ - start_)); |
||||
} |
||||
|
||||
// Return an internal key (suitable for passing to an internal iterator)
|
||||
Slice internal_key() const { |
||||
return Slice(kstart_, static_cast<size_t>(end_ - kstart_)); |
||||
} |
||||
|
||||
// Return the user key
|
||||
Slice user_key() const { |
||||
return Slice(kstart_, static_cast<size_t>(end_ - kstart_ - 8)); |
||||
} |
||||
|
||||
private: |
||||
// We construct a char array of the form:
|
||||
// klength varint32 <-- start_
|
||||
// userkey char[klength] <-- kstart_
|
||||
// tag uint64
|
||||
// <-- end_
|
||||
// The array is a suitable MemTable key.
|
||||
// The suffix starting with "userkey" can be used as an InternalKey.
|
||||
const char* start_; |
||||
const char* kstart_; |
||||
const char* end_; |
||||
char space_[200]; // Avoid allocation for short keys
|
||||
|
||||
// No copying allowed
|
||||
LookupKey(const LookupKey&); |
||||
void operator=(const LookupKey&); |
||||
}; |
||||
|
||||
inline LookupKey::~LookupKey() { |
||||
if (start_ != space_) delete[] start_; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,249 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#pragma once |
||||
#include <algorithm> |
||||
#include <string> |
||||
#include "db/lookup_key.h" |
||||
#include "db/merge_context.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/statistics.h" |
||||
#include "rocksdb/types.h" |
||||
#include "util/autovector.h" |
||||
|
||||
namespace rocksdb { |
||||
class GetContext; |
||||
|
||||
struct KeyContext { |
||||
const Slice* key; |
||||
LookupKey* lkey; |
||||
Slice ukey; |
||||
Slice ikey; |
||||
Status* s; |
||||
MergeContext merge_context; |
||||
SequenceNumber max_covering_tombstone_seq; |
||||
bool key_exists; |
||||
SequenceNumber seq; |
||||
void* cb_arg; |
||||
PinnableSlice* value; |
||||
GetContext* get_context; |
||||
|
||||
KeyContext(const Slice& user_key, PinnableSlice* val, Status* stat) |
||||
: key(&user_key), |
||||
lkey(nullptr), |
||||
s(stat), |
||||
max_covering_tombstone_seq(0), |
||||
key_exists(false), |
||||
seq(0), |
||||
cb_arg(nullptr), |
||||
value(val), |
||||
get_context(nullptr) {} |
||||
|
||||
KeyContext() = default; |
||||
}; |
||||
|
||||
// The MultiGetContext class is a container for the sorted list of keys that
|
||||
// we need to lookup in a batch. Its main purpose is to make batch execution
|
||||
// easier by allowing various stages of the MultiGet lookups to operate on
|
||||
// subsets of keys, potentially non-contiguous. In order to accomplish this,
|
||||
// it defines the following classes -
|
||||
//
|
||||
// MultiGetContext::Range
|
||||
// MultiGetContext::Range::Iterator
|
||||
// MultiGetContext::Range::IteratorWrapper
|
||||
//
|
||||
// Here is an example of how this can be used -
|
||||
//
|
||||
// {
|
||||
// MultiGetContext ctx(...);
|
||||
// MultiGetContext::Range range = ctx.GetMultiGetRange();
|
||||
//
|
||||
// // Iterate to determine some subset of the keys
|
||||
// MultiGetContext::Range::Iterator start = range.begin();
|
||||
// MultiGetContext::Range::Iterator end = ...;
|
||||
//
|
||||
// // Make a new range with a subset of keys
|
||||
// MultiGetContext::Range subrange(range, start, end);
|
||||
//
|
||||
// // Define an auxillary vector, if needed, to hold additional data for
|
||||
// // each key
|
||||
// std::array<Foo, MultiGetContext::MAX_BATCH_SIZE> aux;
|
||||
//
|
||||
// // Iterate over the subrange and the auxillary vector simultaneously
|
||||
// MultiGetContext::Range::Iterator iter = subrange.begin();
|
||||
// for (; iter != subrange.end(); ++iter) {
|
||||
// KeyContext& key = *iter;
|
||||
// Foo& aux_key = aux_iter[iter.index()];
|
||||
// ...
|
||||
// }
|
||||
// }
|
||||
class MultiGetContext { |
||||
public: |
||||
// Limit the number of keys in a batch to this number. Benchmarks show that
|
||||
// there is negligible benefit for batches exceeding this. Keeping this < 64
|
||||
// simplifies iteration, as well as reduces the amount of stack allocations
|
||||
// htat need to be performed
|
||||
static const int MAX_BATCH_SIZE = 32; |
||||
|
||||
MultiGetContext(KeyContext** sorted_keys, size_t num_keys, |
||||
SequenceNumber snapshot) |
||||
: sorted_keys_(sorted_keys), |
||||
num_keys_(num_keys), |
||||
value_mask_(0), |
||||
lookup_key_ptr_(reinterpret_cast<LookupKey*>(lookup_key_stack_buf)) { |
||||
int index = 0; |
||||
|
||||
if (num_keys > MAX_LOOKUP_KEYS_ON_STACK) { |
||||
lookup_key_heap_buf.reset(new char[sizeof(LookupKey) * num_keys]); |
||||
lookup_key_ptr_ = reinterpret_cast<LookupKey*>( |
||||
lookup_key_heap_buf.get()); |
||||
} |
||||
|
||||
for (size_t iter = 0; iter != num_keys_; ++iter) { |
||||
sorted_keys_[iter]->lkey = new (&lookup_key_ptr_[index]) |
||||
LookupKey(*sorted_keys_[iter]->key, snapshot); |
||||
sorted_keys_[iter]->ukey = sorted_keys_[iter]->lkey->user_key(); |
||||
sorted_keys_[iter]->ikey = sorted_keys_[iter]->lkey->internal_key(); |
||||
index++; |
||||
} |
||||
} |
||||
|
||||
~MultiGetContext() { |
||||
for (size_t i = 0; i < num_keys_; ++i) { |
||||
lookup_key_ptr_[i].~LookupKey(); |
||||
} |
||||
} |
||||
|
||||
private: |
||||
static const int MAX_LOOKUP_KEYS_ON_STACK = 16; |
||||
alignas(alignof(LookupKey)) |
||||
char lookup_key_stack_buf[sizeof(LookupKey) * MAX_LOOKUP_KEYS_ON_STACK]; |
||||
KeyContext** sorted_keys_; |
||||
size_t num_keys_; |
||||
uint64_t value_mask_; |
||||
std::unique_ptr<char> lookup_key_heap_buf; |
||||
LookupKey* lookup_key_ptr_; |
||||
|
||||
public: |
||||
// MultiGetContext::Range - Specifies a range of keys, by start and end index,
|
||||
// from the parent MultiGetContext. Each range contains a bit vector that
|
||||
// indicates whether the corresponding keys need to be processed or skipped.
|
||||
// A Range object can be copy constructed, and the new object inherits the
|
||||
// original Range's bit vector. This is useful for progressively skipping
|
||||
// keys as the lookup goes through various stages. For example, when looking
|
||||
// up keys in the same SST file, a Range is created excluding keys not
|
||||
// belonging to that file. A new Range is then copy constructed and individual
|
||||
// keys are skipped based on bloom filter lookup.
|
||||
class Range { |
||||
public: |
||||
// MultiGetContext::Range::Iterator - A forward iterator that iterates over
|
||||
// non-skippable keys in a Range, as well as keys whose final value has been
|
||||
// found. The latter is tracked by MultiGetContext::value_mask_
|
||||
class Iterator { |
||||
public: |
||||
// -- iterator traits
|
||||
typedef Iterator self_type; |
||||
typedef KeyContext value_type; |
||||
typedef KeyContext& reference; |
||||
typedef KeyContext* pointer; |
||||
typedef int difference_type; |
||||
typedef std::forward_iterator_tag iterator_category; |
||||
|
||||
Iterator(const Range* range, size_t idx) |
||||
: range_(range), ctx_(range->ctx_), index_(idx) { |
||||
while (index_ < range_->end_ && |
||||
(1ull << index_) & |
||||
(range_->ctx_->value_mask_ | range_->skip_mask_)) |
||||
index_++; |
||||
} |
||||
|
||||
Iterator(const Iterator&) = default; |
||||
Iterator& operator=(const Iterator&) = default; |
||||
|
||||
Iterator& operator++() { |
||||
while (++index_ < range_->end_ && |
||||
(1ull << index_) & |
||||
(range_->ctx_->value_mask_ | range_->skip_mask_)) |
||||
; |
||||
return *this; |
||||
} |
||||
|
||||
bool operator==(Iterator other) const { |
||||
assert(range_->ctx_ == other.range_->ctx_); |
||||
return index_ == other.index_; |
||||
} |
||||
|
||||
bool operator!=(Iterator other) const { |
||||
assert(range_->ctx_ == other.range_->ctx_); |
||||
return index_ != other.index_; |
||||
} |
||||
|
||||
KeyContext& operator*() { |
||||
assert(index_ < range_->end_ && index_ >= range_->start_); |
||||
return *(ctx_->sorted_keys_[index_]); |
||||
} |
||||
|
||||
KeyContext* operator->() { |
||||
assert(index_ < range_->end_ && index_ >= range_->start_); |
||||
return ctx_->sorted_keys_[index_]; |
||||
} |
||||
|
||||
size_t index() { return index_; } |
||||
|
||||
private: |
||||
friend Range; |
||||
const Range* range_; |
||||
const MultiGetContext* ctx_; |
||||
size_t index_; |
||||
}; |
||||
|
||||
Range(const Range& mget_range, |
||||
const Iterator& first, |
||||
const Iterator& last) { |
||||
ctx_ = mget_range.ctx_; |
||||
start_ = first.index_; |
||||
end_ = last.index_; |
||||
skip_mask_ = mget_range.skip_mask_; |
||||
} |
||||
|
||||
Range() = default; |
||||
|
||||
Iterator begin() const { return Iterator(this, start_); } |
||||
|
||||
Iterator end() const { return Iterator(this, end_); } |
||||
|
||||
bool empty() { |
||||
return (((1ull << end_) - 1) & ~((1ull << start_) - 1) & |
||||
~(ctx_->value_mask_ | skip_mask_)) == 0; |
||||
} |
||||
|
||||
void SkipKey(const Iterator& iter) { skip_mask_ |= 1ull << iter.index_; } |
||||
|
||||
// Update the value_mask_ in MultiGetContext so its
|
||||
// immediately reflected in all the Range Iterators
|
||||
void MarkKeyDone(Iterator& iter) { |
||||
ctx_->value_mask_ |= (1ull << iter.index_); |
||||
} |
||||
|
||||
bool CheckKeyDone(Iterator& iter) { |
||||
return ctx_->value_mask_ & (1ull << iter.index_); |
||||
} |
||||
|
||||
private: |
||||
friend MultiGetContext; |
||||
MultiGetContext* ctx_; |
||||
size_t start_; |
||||
size_t end_; |
||||
uint64_t skip_mask_; |
||||
|
||||
Range(MultiGetContext* ctx, size_t num_keys) |
||||
: ctx_(ctx), start_(0), end_(num_keys), skip_mask_(0) {} |
||||
}; |
||||
|
||||
// Return the initial range that encompasses all the keys in the batch
|
||||
Range GetMultiGetRange() { return Range(this, num_keys_); } |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
Loading…
Reference in new issue