Summary: Here are the major changes to the Merge Interface. It has been expanded to handle cases where the MergeOperator is not associative. It does so by stacking up merge operations while scanning through the key history (i.e.: during Get() or Compaction), until a valid Put/Delete/end-of-history is encountered; it then applies all of the merge operations in the correct sequence starting with the base/sentinel value. I have also introduced an "AssociativeMerge" function which allows the user to take advantage of associative merge operations (such as in the case of counters). The implementation will always attempt to merge the operations/operands themselves together when they are encountered, and will resort to the "stacking" method if and only if the "associative-merge" fails. This implementation is conjectured to allow MergeOperator to handle the general case, while still providing the user with the ability to take advantage of certain efficiencies in their own merge-operator / data-structure. NOTE: This is a preliminary diff. This must still go through a lot of review, revision, and testing. Feedback welcome! Test Plan: -This is a preliminary diff. I have only just begun testing/debugging it. -I will be testing this with the existing MergeOperator use-cases and unit-tests (counters, string-append, and redis-lists) -I will be "desk-checking" and walking through the code with the help gdb. -I will find a way of stress-testing the new interface / implementation using db_bench, db_test, merge_test, and/or db_stress. -I will ensure that my tests cover all cases: Get-Memtable, Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0, Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found, end-of-history, end-of-file, etc. -A lot of feedback from the reviewers. Reviewers: haobo, dhruba, zshao, emayanke Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11499main
Deon Nicholas
11 years ago
parent
73f9518b21
commit
c2d7826ced
@ -0,0 +1,51 @@ |
||||
/**
|
||||
* Back-end implementation details specific to the Merge Operator. |
||||
* |
||||
* @author Deon Nicholas (dnicholas@fb.com) |
||||
* Copyright 2013 Facebook |
||||
*/ |
||||
|
||||
#include "leveldb/merge_operator.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
// Given a "real" merge from the library, call the user's
|
||||
// associative merge function one-by-one on each of the operands.
|
||||
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
||||
bool AssociativeMergeOperator::Merge( |
||||
const Slice& key, |
||||
const Slice* existing_value, |
||||
const std::deque<std::string>& operand_list, |
||||
std::string* new_value, |
||||
Logger* logger) const { |
||||
|
||||
// Simply loop through the operands
|
||||
Slice temp_existing; |
||||
std::string temp_value; |
||||
for (const auto& operand : operand_list) { |
||||
Slice value(operand); |
||||
if (!Merge(key, existing_value, value, &temp_value, logger)) { |
||||
return false; |
||||
} |
||||
swap(temp_value, *new_value); |
||||
temp_existing = Slice(*new_value); |
||||
existing_value = &temp_existing; |
||||
} |
||||
|
||||
// The result will be in *new_value. All merges succeeded.
|
||||
return true; |
||||
} |
||||
|
||||
// Call the user defined simple merge on the operands;
|
||||
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
||||
bool AssociativeMergeOperator::PartialMerge( |
||||
const Slice& key, |
||||
const Slice& left_operand, |
||||
const Slice& right_operand, |
||||
std::string* new_value, |
||||
Logger* logger) const { |
||||
|
||||
return Merge(key, &left_operand, right_operand, new_value, logger); |
||||
} |
||||
|
||||
} // namespace leveldb
|
||||
@ -0,0 +1,90 @@ |
||||
/**
|
||||
* @author Deon Nicholas (dnicholas@fb.com) |
||||
* Copyright 2013 Facebook |
||||
*/ |
||||
|
||||
#include "stringappend2.h" |
||||
|
||||
#include <memory> |
||||
#include <assert.h> |
||||
|
||||
#include "leveldb/slice.h" |
||||
#include "leveldb/merge_operator.h" |
||||
#include "utilities/merge_operators.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
// Constructor: also specify the delimiter character.
|
||||
StringAppendTESTOperator::StringAppendTESTOperator(char delim_char) |
||||
: delim_(delim_char) { |
||||
} |
||||
|
||||
// Implementation for the merge operation (concatenates two strings)
|
||||
bool StringAppendTESTOperator::Merge(const Slice& key, |
||||
const Slice* existing_value, |
||||
const std::deque<std::string>& operands, |
||||
std::string* new_value, |
||||
Logger* logger) const { |
||||
|
||||
// Clear the *new_value for writing.
|
||||
assert(new_value); |
||||
new_value->clear(); |
||||
|
||||
// Compute the space needed for the final result.
|
||||
int numBytes = 0; |
||||
for(auto it = operands.begin(); it != operands.end(); ++it) { |
||||
numBytes += it->size() + 1; // Plus 1 for the delimiter
|
||||
} |
||||
|
||||
// Only print the delimiter after the first entry has been printed
|
||||
bool printDelim = false; |
||||
|
||||
// Prepend the *existing_value if one exists.
|
||||
if (existing_value) { |
||||
new_value->reserve(numBytes + existing_value->size()); |
||||
new_value->append(existing_value->data(), existing_value->size()); |
||||
printDelim = true; |
||||
} else if (numBytes) { |
||||
new_value->reserve(numBytes-1); // Minus 1 since we have one less delimiter
|
||||
} |
||||
|
||||
// Concatenate the sequence of strings (and add a delimiter between each)
|
||||
for(auto it = operands.begin(); it != operands.end(); ++it) { |
||||
if (printDelim) { |
||||
new_value->append(1,delim_); |
||||
} |
||||
new_value->append(*it); |
||||
printDelim = true; |
||||
} |
||||
|
||||
return true; |
||||
} |
||||
|
||||
bool StringAppendTESTOperator::PartialMerge(const Slice& key, |
||||
const Slice& left_operand, |
||||
const Slice& right_operand, |
||||
std::string* new_value, |
||||
Logger* logger) const { |
||||
return false; |
||||
|
||||
// // Clear the *new_value for writing.
|
||||
// assert(new_value);
|
||||
// new_value->clear();
|
||||
//
|
||||
// // Generic append
|
||||
// // Reserve correct size for *new_value, and apply concatenation.
|
||||
// new_value->reserve(left_operand.size() + 1 + right_operand.size());
|
||||
// new_value->assign(left_operand.data(), left_operand.size());
|
||||
// new_value->append(1,delim_);
|
||||
// new_value->append(right_operand.data(), right_operand.size());
|
||||
//
|
||||
// return true;
|
||||
|
||||
} |
||||
|
||||
const char* StringAppendTESTOperator::Name() const { |
||||
return "StringAppendTESTOperator"; |
||||
} |
||||
|
||||
} // namespace leveldb
|
||||
|
||||
@ -0,0 +1,42 @@ |
||||
/**
|
||||
* A TEST MergeOperator for rocksdb/leveldb that implements string append. |
||||
* It is built using the MergeOperator interface rather than the simpler |
||||
* AssociativeMergeOperator interface. This is useful for testing/benchmarking. |
||||
* While the two operators are semantically the same, all production code |
||||
* should use the StringAppendOperator defined in stringappend.{h,cc}. The |
||||
* operator defined in the present file is primarily for testing. |
||||
* |
||||
* @author Deon Nicholas (dnicholas@fb.com) |
||||
* Copyright 2013 Facebook |
||||
*/ |
||||
|
||||
#include "leveldb/merge_operator.h" |
||||
#include "leveldb/slice.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
class StringAppendTESTOperator : public MergeOperator { |
||||
public: |
||||
|
||||
StringAppendTESTOperator(char delim_char); /// Constructor with delimiter
|
||||
|
||||
virtual bool Merge(const Slice& key, |
||||
const Slice* existing_value, |
||||
const std::deque<std::string>& operand_sequence, |
||||
std::string* new_value, |
||||
Logger* logger) const override; |
||||
|
||||
virtual bool PartialMerge(const Slice& key, |
||||
const Slice& left_operand, |
||||
const Slice& right_operand, |
||||
std::string* new_value, |
||||
Logger* logger) const override; |
||||
|
||||
virtual const char* Name() const override; |
||||
|
||||
private: |
||||
char delim_; // The delimiter is inserted between elements
|
||||
|
||||
}; |
||||
|
||||
} // namespace leveldb
|
||||
Loading…
Reference in new issue