NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Eliminate std::deque initialization while iterating over merge operands

Browse Source 
Summary:
This patch is similar to D52563, When we iterate over a DB with merge operands we keep creating std::queue to store the operands, optimize this by reusing merge_operands_ data member

Before the patch

```
./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq" --db="/dev/shm/bench_merge_memcpy_on_the_fly/" --merge_operator="put" --merge_keys=10000 --num=10000

DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
mergerandom  :       3.757 micros/op 266141 ops/sec;   29.4 MB/s ( updates:10000)
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.413 micros/op 2423538 ops/sec;  268.1 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.451 micros/op 2219071 ops/sec;  245.5 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.420 micros/op 2382039 ops/sec;  263.5 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.408 micros/op 2452017 ops/sec;  271.3 MB/s

DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
mergerandom  :       3.947 micros/op 253376 ops/sec;   28.0 MB/s ( updates:10000)
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.441 micros/op 2266473 ops/sec;  250.7 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.471 micros/op 2122033 ops/sec;  234.8 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.440 micros/op 2271407 ops/sec;  251.3 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.429 micros/op 2331471 ops/sec;  257.9 MB/s
```

with the patch

```
./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq" --db="/dev/shm/bench_merge_memcpy_on_the_fly/" --merge_operator="put" --merge_keys=10000 --num=10000

DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
mergerandom  :       4.080 micros/op 245092 ops/sec;   27.1 MB/s ( updates:10000)
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.308 micros/op 3241843 ops/sec;  358.6 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.312 micros/op 3200408 ops/sec;  354.0 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.332 micros/op 3013962 ops/sec;  333.4 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.300 micros/op 3328017 ops/sec;  368.2 MB/s

DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
mergerandom  :       3.973 micros/op 251705 ops/sec;   27.8 MB/s ( updates:10000)
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.320 micros/op 3123752 ops/sec;  345.6 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.335 micros/op 2986641 ops/sec;  330.4 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.339 micros/op 2950047 ops/sec;  326.4 MB/s
DB path: [/dev/shm/bench_merge_memcpy_on_the_fly/]
readseq      :       0.319 micros/op 3131565 ops/sec;  346.4 MB/s
```

Test Plan: make check -j64

Reviewers: yhchiang, andrewkr, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D56031
main
Islam AbdelRahman 9 years ago
parent f38540b12a
commit 8a1a603fdb
2 changed files with 30 additions and 20 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 45
      db/db_iter.cc
  2. 5
      db/merge_context.h

45
db/db_iter.cc
Unescape View File

@ -15,6 +15,7 @@
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/filename.h" #include "db/filename.h"
#include "db/merge_context.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/iterator.h" #include "rocksdb/iterator.h"
@ -251,7 +252,7 @@ class DBIter: public Iterator {
bool prefix_same_as_start_; bool prefix_same_as_start_;
bool iter_pinned_; bool iter_pinned_;
// List of operands for merge operator. // List of operands for merge operator.
std::deque<std::string> merge_operands_; MergeContext merge_context_;
LocalStatistics local_stats_; LocalStatistics local_stats_;
// No copying allowed // No copying allowed
@ -411,9 +412,9 @@ void DBIter::MergeValuesNewToOld() {
return; return;
} }
merge_context_.Clear();
// Start the merge process by pushing the first operand // Start the merge process by pushing the first operand
std::deque<std::string> operands; merge_context_.PushOperand(iter_->value());
operands.push_front(iter_->value().ToString());
ParsedInternalKey ikey; ParsedInternalKey ikey;
for (iter_->Next(); iter_->Valid(); iter_->Next()) { for (iter_->Next(); iter_->Valid(); iter_->Next()) {
@ -438,7 +439,8 @@ void DBIter::MergeValuesNewToOld() {
{ {
StopWatchNano timer(env_, statistics_ != nullptr); StopWatchNano timer(env_, statistics_ != nullptr);
PERF_TIMER_GUARD(merge_operator_time_nanos); PERF_TIMER_GUARD(merge_operator_time_nanos);
user_merge_operator_->FullMerge(ikey.user_key, &val, operands, user_merge_operator_->FullMerge(ikey.user_key, &val,
merge_context_.GetOperands(),
&saved_value_, logger_); &saved_value_, logger_);
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
timer.ElapsedNanos()); timer.ElapsedNanos());
@ -450,7 +452,7 @@ void DBIter::MergeValuesNewToOld() {
// hit a merge, add the value as an operand and run associative merge. // hit a merge, add the value as an operand and run associative merge.
// when complete, add result to operands and continue. // when complete, add result to operands and continue.
const Slice& val = iter_->value(); const Slice& val = iter_->value();
operands.push_front(val.ToString()); merge_context_.PushOperand(val);
} else { } else {
assert(false); assert(false);
} }
@ -463,8 +465,9 @@ void DBIter::MergeValuesNewToOld() {
// a deletion marker. // a deletion marker.
// feed null as the existing value to the merge operator, such that // feed null as the existing value to the merge operator, such that
// client can differentiate this scenario and do things accordingly. // client can differentiate this scenario and do things accordingly.
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands, user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
&saved_value_, logger_); merge_context_.GetOperands(), &saved_value_,
logger_);
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos()); RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
} }
} }
@ -556,7 +559,7 @@ void DBIter::PrevInternal() {
// saved_value_ // saved_value_
bool DBIter::FindValueForCurrentKey() { bool DBIter::FindValueForCurrentKey() {
assert(iter_->Valid()); assert(iter_->Valid());
merge_operands_.clear(); merge_context_.Clear();
// last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or // last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
// kTypeValue) // kTypeValue)
ValueType last_not_merge_type = kTypeDeletion; ValueType last_not_merge_type = kTypeDeletion;
@ -576,19 +579,19 @@ bool DBIter::FindValueForCurrentKey() {
last_key_entry_type = ikey.type; last_key_entry_type = ikey.type;
switch (last_key_entry_type) { switch (last_key_entry_type) {
case kTypeValue: case kTypeValue:
merge_operands_.clear(); merge_context_.Clear();
saved_value_ = iter_->value().ToString(); saved_value_ = iter_->value().ToString();
last_not_merge_type = kTypeValue; last_not_merge_type = kTypeValue;
break; break;
case kTypeDeletion: case kTypeDeletion:
case kTypeSingleDeletion: case kTypeSingleDeletion:
merge_operands_.clear(); merge_context_.Clear();
last_not_merge_type = last_key_entry_type; last_not_merge_type = last_key_entry_type;
PERF_COUNTER_ADD(internal_delete_skipped_count, 1); PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
break; break;
case kTypeMerge: case kTypeMerge:
assert(user_merge_operator_ != nullptr); assert(user_merge_operator_ != nullptr);
merge_operands_.push_back(iter_->value().ToString()); merge_context_.PushOperandBack(iter_->value());
break; break;
default: default:
assert(false); assert(false);
@ -611,8 +614,8 @@ bool DBIter::FindValueForCurrentKey() {
StopWatchNano timer(env_, statistics_ != nullptr); StopWatchNano timer(env_, statistics_ != nullptr);
PERF_TIMER_GUARD(merge_operator_time_nanos); PERF_TIMER_GUARD(merge_operator_time_nanos);
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
merge_operands_, &saved_value_, merge_context_.GetOperands(),
logger_); &saved_value_, logger_);
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
timer.ElapsedNanos()); timer.ElapsedNanos());
} else { } else {
@ -623,8 +626,8 @@ bool DBIter::FindValueForCurrentKey() {
StopWatchNano timer(env_, statistics_ != nullptr); StopWatchNano timer(env_, statistics_ != nullptr);
PERF_TIMER_GUARD(merge_operator_time_nanos); PERF_TIMER_GUARD(merge_operator_time_nanos);
user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice, user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice,
merge_operands_, &saved_value_, merge_context_.GetOperands(),
logger_); &saved_value_, logger_);
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
timer.ElapsedNanos()); timer.ElapsedNanos());
} }
@ -667,11 +670,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
// kTypeMerge. We need to collect all kTypeMerge values and save them // kTypeMerge. We need to collect all kTypeMerge values and save them
// in operands // in operands
std::deque<std::string> operands; merge_context_.Clear();
while (iter_->Valid() && while (iter_->Valid() &&
user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) && user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
ikey.type == kTypeMerge) { ikey.type == kTypeMerge) {
operands.push_front(iter_->value().ToString()); merge_context_.PushOperand(iter_->value());
iter_->Next(); iter_->Next();
FindParseableKey(&ikey, kForward); FindParseableKey(&ikey, kForward);
} }
@ -682,7 +685,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
{ {
StopWatchNano timer(env_, statistics_ != nullptr); StopWatchNano timer(env_, statistics_ != nullptr);
PERF_TIMER_GUARD(merge_operator_time_nanos); PERF_TIMER_GUARD(merge_operator_time_nanos);
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands, user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
merge_context_.GetOperands(),
&saved_value_, logger_); &saved_value_, logger_);
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos()); RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
} }
@ -700,8 +704,9 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
{ {
StopWatchNano timer(env_, statistics_ != nullptr); StopWatchNano timer(env_, statistics_ != nullptr);
PERF_TIMER_GUARD(merge_operator_time_nanos); PERF_TIMER_GUARD(merge_operator_time_nanos);
user_merge_operator_->FullMerge(saved_key_.GetKey(), &val, operands, user_merge_operator_->FullMerge(saved_key_.GetKey(), &val,
&saved_value_, logger_); merge_context_.GetOperands(), &saved_value_,
logger_);
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos()); RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
} }
valid_ = true; valid_ = true;

5
db/merge_context.h
Unescape View File

@ -37,6 +37,11 @@ public:
Initialize(); Initialize();
operand_list->push_front(operand_slice.ToString()); operand_list->push_front(operand_slice.ToString());
} }
// Push back a merge operand
void PushOperandBack(const Slice& operand_slice) {
Initialize();
operand_list->push_back(operand_slice.ToString());
}
// return total number of operands in the list // return total number of operands in the list
size_t GetNumOperands() const { size_t GetNumOperands() const {
if (!operand_list) { if (!operand_list) {

Write Preview
Loading…
Cancel
Save