|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#pragma once
|
|
|
|
#include <string>
|
|
|
|
#include <memory>
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
12 years ago
|
|
|
#include <deque>
|
|
|
|
#include "db/dbformat.h"
|
|
|
|
#include "db/skiplist.h"
|
|
|
|
#include "db/version_set.h"
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
#include "rocksdb/memtablerep.h"
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
12 years ago
|
|
|
#include "util/arena_impl.h"
|
|
|
|
#include "util/dynamic_bloom.h"
|
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
class Mutex;
|
|
|
|
class MemTableIterator;
|
|
|
|
class MergeContext;
|
|
|
|
|
|
|
|
class MemTable {
|
|
|
|
public:
|
|
|
|
struct KeyComparator : public MemTableRep::KeyComparator {
|
|
|
|
const InternalKeyComparator comparator;
|
|
|
|
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
|
|
|
|
virtual int operator()(const char* prefix_len_key1,
|
|
|
|
const char* prefix_len_key2) const;
|
|
|
|
virtual int operator()(const char* prefix_len_key,
|
|
|
|
const Slice& key) const override;
|
|
|
|
};
|
|
|
|
|
|
|
|
// MemTables are reference counted. The initial reference count
|
|
|
|
// is zero and the caller must call Ref() at least once.
|
|
|
|
explicit MemTable(const InternalKeyComparator& comparator,
|
|
|
|
const Options& options = Options());
|
|
|
|
|
|
|
|
~MemTable();
|
|
|
|
|
|
|
|
// Increase reference count.
|
|
|
|
void Ref() { ++refs_; }
|
|
|
|
|
|
|
|
// Drop reference count.
|
|
|
|
// If the refcount goes to zero return this memtable, otherwise return null
|
|
|
|
MemTable* Unref() {
|
|
|
|
--refs_;
|
|
|
|
assert(refs_ >= 0);
|
|
|
|
if (refs_ <= 0) {
|
|
|
|
return this;
|
|
|
|
}
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Returns an estimate of the number of bytes of data in use by this
|
|
|
|
// data structure.
|
|
|
|
//
|
|
|
|
// REQUIRES: external synchronization to prevent simultaneous
|
|
|
|
// operations on the same MemTable.
|
|
|
|
size_t ApproximateMemoryUsage();
|
|
|
|
|
|
|
|
// Return an iterator that yields the contents of the memtable.
|
|
|
|
//
|
|
|
|
// The caller must ensure that the underlying MemTable remains live
|
|
|
|
// while the returned iterator is live. The keys returned by this
|
|
|
|
// iterator are internal keys encoded by AppendInternalKey in the
|
|
|
|
// db/dbformat.{h,cc} module.
|
|
|
|
//
|
|
|
|
// If options.prefix is supplied, it is passed to the underlying MemTableRep
|
|
|
|
// as a hint that the iterator only need to support access to keys with that
|
|
|
|
// specific prefix.
|
|
|
|
// If options.prefix is not supplied and options.prefix_seek is set, the
|
|
|
|
// iterator is not bound to a specific prefix. However, the semantics of
|
|
|
|
// Seek is changed - the result might only include keys with the same prefix
|
|
|
|
// as the seek-key.
|
|
|
|
Iterator* NewIterator(const ReadOptions& options = ReadOptions());
|
|
|
|
|
|
|
|
// Add an entry into memtable that maps key to value at the
|
|
|
|
// specified sequence number and with the specified type.
|
|
|
|
// Typically value will be empty if type==kTypeDeletion.
|
|
|
|
void Add(SequenceNumber seq, ValueType type,
|
|
|
|
const Slice& key,
|
|
|
|
const Slice& value);
|
|
|
|
|
|
|
|
// If memtable contains a value for key, store it in *value and return true.
|
|
|
|
// If memtable contains a deletion for key, store a NotFound() error
|
|
|
|
// in *status and return true.
|
|
|
|
// If memtable contains Merge operation as the most recent entry for a key,
|
|
|
|
// and the merge process does not stop (not reaching a value or delete),
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
12 years ago
|
|
|
// prepend the current merge operand to *operands.
|
|
|
|
// store MergeInProgress in s, and return false.
|
|
|
|
// Else, return false.
|
|
|
|
bool Get(const LookupKey& key, std::string* value, Status* s,
|
|
|
|
MergeContext& merge_context, const Options& options);
|
|
|
|
|
|
|
|
// Attempts to update the new_value inplace, else does normal Add
|
|
|
|
// Pseudocode
|
|
|
|
// if key exists in current memtable && prev_value is of type kTypeValue
|
|
|
|
// if new sizeof(new_value) <= sizeof(prev_value)
|
|
|
|
// update inplace
|
|
|
|
// else add(key, new_value)
|
|
|
|
// else add(key, new_value)
|
|
|
|
void Update(SequenceNumber seq,
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
12 years ago
|
|
|
const Slice& key,
|
|
|
|
const Slice& value);
|
|
|
|
|
|
|
|
// If prev_value for key exits, attempts to update it inplace.
|
|
|
|
// else returns false
|
|
|
|
// Pseudocode
|
|
|
|
// if key exists in current memtable && prev_value is of type kTypeValue
|
|
|
|
// new_value = delta(prev_value)
|
|
|
|
// if sizeof(new_value) <= sizeof(prev_value)
|
|
|
|
// update inplace
|
|
|
|
// else add(key, new_value)
|
|
|
|
// else return false
|
|
|
|
bool UpdateCallback(SequenceNumber seq,
|
|
|
|
const Slice& key,
|
|
|
|
const Slice& delta,
|
|
|
|
const Options& options);
|
|
|
|
|
|
|
|
// Returns the number of successive merge entries starting from the newest
|
|
|
|
// entry for the key up to the last non-merge entry or last entry for the
|
|
|
|
// key in the memtable.
|
|
|
|
size_t CountSuccessiveMergeEntries(const LookupKey& key);
|
|
|
|
|
|
|
|
// Returns the edits area that is needed for flushing the memtable
|
|
|
|
VersionEdit* GetEdits() { return &edit_; }
|
|
|
|
|
|
|
|
// Returns the sequence number of the first element that was inserted
|
|
|
|
// into the memtable
|
|
|
|
SequenceNumber GetFirstSequenceNumber() { return first_seqno_; }
|
|
|
|
|
|
|
|
// Returns the next active logfile number when this memtable is about to
|
|
|
|
// be flushed to storage
|
|
|
|
uint64_t GetNextLogNumber() { return mem_next_logfile_number_; }
|
|
|
|
|
|
|
|
// Sets the next active logfile number when this memtable is about to
|
|
|
|
// be flushed to storage
|
|
|
|
void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
|
|
|
|
|
|
|
|
// Returns the logfile number that can be safely deleted when this
|
|
|
|
// memstore is flushed to storage
|
|
|
|
uint64_t GetLogNumber() { return mem_logfile_number_; }
|
|
|
|
|
|
|
|
// Sets the logfile number that can be safely deleted when this
|
|
|
|
// memstore is flushed to storage
|
|
|
|
void SetLogNumber(uint64_t num) { mem_logfile_number_ = num; }
|
|
|
|
|
|
|
|
// Notify the underlying storage that no more items will be added
|
|
|
|
void MarkImmutable() { table_->MarkReadOnly(); }
|
|
|
|
|
|
|
|
private:
|
|
|
|
friend class MemTableIterator;
|
|
|
|
friend class MemTableBackwardIterator;
|
|
|
|
friend class MemTableList;
|
|
|
|
|
|
|
|
KeyComparator comparator_;
|
|
|
|
int refs_;
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
12 years ago
|
|
|
ArenaImpl arena_impl_;
|
|
|
|
unique_ptr<MemTableRep> table_;
|
|
|
|
|
|
|
|
// These are used to manage memtable flushes to storage
|
|
|
|
bool flush_in_progress_; // started the flush
|
|
|
|
bool flush_completed_; // finished the flush
|
|
|
|
uint64_t file_number_; // filled up after flush is complete
|
|
|
|
|
|
|
|
// The updates to be applied to the transaction log when this
|
|
|
|
// memtable is flushed to storage.
|
|
|
|
VersionEdit edit_;
|
|
|
|
|
|
|
|
// The sequence number of the kv that was inserted first
|
|
|
|
SequenceNumber first_seqno_;
|
|
|
|
|
|
|
|
// The log files earlier than this number can be deleted.
|
|
|
|
uint64_t mem_next_logfile_number_;
|
|
|
|
|
|
|
|
// The log file that backs this memtable (to be deleted when
|
|
|
|
// memtable flush is done)
|
|
|
|
uint64_t mem_logfile_number_;
|
|
|
|
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
12 years ago
|
|
|
// rw locks for inplace updates
|
|
|
|
std::vector<port::RWMutex> locks_;
|
|
|
|
|
|
|
|
// No copying allowed
|
|
|
|
MemTable(const MemTable&);
|
|
|
|
void operator=(const MemTable&);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
12 years ago
|
|
|
|
|
|
|
// Get the lock associated for the key
|
|
|
|
port::RWMutex* GetLock(const Slice& key);
|
|
|
|
|
|
|
|
const SliceTransform* const prefix_extractor_;
|
|
|
|
std::unique_ptr<DynamicBloom> prefix_bloom_;
|
|
|
|
};
|
|
|
|
|
|
|
|
extern const char* EncodeKey(std::string* scratch, const Slice& target);
|
|
|
|
|
|
|
|
} // namespace rocksdb
|