Remove v1 RangeDelAggregator (#4778)
Summary: Now that v2 is fully functional, the v1 aggregator is removed. The v2 aggregator has been renamed. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4778 Differential Revision: D13495930 Pulled By: abhimadan fbshipit-source-id: 9d69500a60a283e79b6c4fa938fc68a8aa4d40d6main
Abhishek Madan
6 years ago
committed by
Facebook Github Bot
parent
311cd8cf2f
commit
81b6b09f6b
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,492 +0,0 @@ |
|||||||
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
#include "db/range_del_aggregator_v2.h" |
|
||||||
|
|
||||||
#include "db/compaction_iteration_stats.h" |
|
||||||
#include "db/dbformat.h" |
|
||||||
#include "db/pinned_iterators_manager.h" |
|
||||||
#include "db/range_del_aggregator.h" |
|
||||||
#include "db/range_tombstone_fragmenter.h" |
|
||||||
#include "db/version_edit.h" |
|
||||||
#include "include/rocksdb/comparator.h" |
|
||||||
#include "include/rocksdb/types.h" |
|
||||||
#include "table/internal_iterator.h" |
|
||||||
#include "table/scoped_arena_iterator.h" |
|
||||||
#include "table/table_builder.h" |
|
||||||
#include "util/heap.h" |
|
||||||
#include "util/kv_map.h" |
|
||||||
#include "util/vector_iterator.h" |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
TruncatedRangeDelIterator::TruncatedRangeDelIterator( |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> iter, |
|
||||||
const InternalKeyComparator* icmp, const InternalKey* smallest, |
|
||||||
const InternalKey* largest) |
|
||||||
: iter_(std::move(iter)), |
|
||||||
icmp_(icmp), |
|
||||||
smallest_ikey_(smallest), |
|
||||||
largest_ikey_(largest) { |
|
||||||
if (smallest != nullptr) { |
|
||||||
pinned_bounds_.emplace_back(); |
|
||||||
auto& parsed_smallest = pinned_bounds_.back(); |
|
||||||
if (!ParseInternalKey(smallest->Encode(), &parsed_smallest)) { |
|
||||||
assert(false); |
|
||||||
} |
|
||||||
smallest_ = &parsed_smallest; |
|
||||||
} |
|
||||||
if (largest != nullptr) { |
|
||||||
pinned_bounds_.emplace_back(); |
|
||||||
auto& parsed_largest = pinned_bounds_.back(); |
|
||||||
if (!ParseInternalKey(largest->Encode(), &parsed_largest)) { |
|
||||||
assert(false); |
|
||||||
} |
|
||||||
if (parsed_largest.type == kTypeRangeDeletion && |
|
||||||
parsed_largest.sequence == kMaxSequenceNumber) { |
|
||||||
// The file boundary has been artificially extended by a range tombstone.
|
|
||||||
// We do not need to adjust largest to properly truncate range
|
|
||||||
// tombstones that extend past the boundary.
|
|
||||||
} else if (parsed_largest.sequence == 0) { |
|
||||||
// The largest key in the sstable has a sequence number of 0. Since we
|
|
||||||
// guarantee that no internal keys with the same user key and sequence
|
|
||||||
// number can exist in a DB, we know that the largest key in this sstable
|
|
||||||
// cannot exist as the smallest key in the next sstable. This further
|
|
||||||
// implies that no range tombstone in this sstable covers largest;
|
|
||||||
// otherwise, the file boundary would have been artificially extended.
|
|
||||||
//
|
|
||||||
// Therefore, we will never truncate a range tombstone at largest, so we
|
|
||||||
// can leave it unchanged.
|
|
||||||
} else { |
|
||||||
// The same user key may straddle two sstable boundaries. To ensure that
|
|
||||||
// the truncated end key can cover the largest key in this sstable, reduce
|
|
||||||
// its sequence number by 1.
|
|
||||||
parsed_largest.sequence -= 1; |
|
||||||
} |
|
||||||
largest_ = &parsed_largest; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
bool TruncatedRangeDelIterator::Valid() const { |
|
||||||
return iter_->Valid() && |
|
||||||
(smallest_ == nullptr || |
|
||||||
icmp_->Compare(*smallest_, iter_->parsed_end_key()) < 0) && |
|
||||||
(largest_ == nullptr || |
|
||||||
icmp_->Compare(iter_->parsed_start_key(), *largest_) < 0); |
|
||||||
} |
|
||||||
|
|
||||||
void TruncatedRangeDelIterator::Next() { iter_->TopNext(); } |
|
||||||
|
|
||||||
void TruncatedRangeDelIterator::Prev() { iter_->TopPrev(); } |
|
||||||
|
|
||||||
void TruncatedRangeDelIterator::InternalNext() { iter_->Next(); } |
|
||||||
|
|
||||||
// NOTE: target is a user key
|
|
||||||
void TruncatedRangeDelIterator::Seek(const Slice& target) { |
|
||||||
if (largest_ != nullptr && |
|
||||||
icmp_->Compare(*largest_, ParsedInternalKey(target, kMaxSequenceNumber, |
|
||||||
kTypeRangeDeletion)) <= 0) { |
|
||||||
iter_->Invalidate(); |
|
||||||
return; |
|
||||||
} |
|
||||||
if (smallest_ != nullptr && |
|
||||||
icmp_->user_comparator()->Compare(target, smallest_->user_key) < 0) { |
|
||||||
iter_->Seek(smallest_->user_key); |
|
||||||
return; |
|
||||||
} |
|
||||||
iter_->Seek(target); |
|
||||||
} |
|
||||||
|
|
||||||
// NOTE: target is a user key
|
|
||||||
void TruncatedRangeDelIterator::SeekForPrev(const Slice& target) { |
|
||||||
if (smallest_ != nullptr && |
|
||||||
icmp_->Compare(ParsedInternalKey(target, 0, kTypeRangeDeletion), |
|
||||||
*smallest_) < 0) { |
|
||||||
iter_->Invalidate(); |
|
||||||
return; |
|
||||||
} |
|
||||||
if (largest_ != nullptr && |
|
||||||
icmp_->user_comparator()->Compare(largest_->user_key, target) < 0) { |
|
||||||
iter_->SeekForPrev(largest_->user_key); |
|
||||||
return; |
|
||||||
} |
|
||||||
iter_->SeekForPrev(target); |
|
||||||
} |
|
||||||
|
|
||||||
void TruncatedRangeDelIterator::SeekToFirst() { |
|
||||||
if (smallest_ != nullptr) { |
|
||||||
iter_->Seek(smallest_->user_key); |
|
||||||
return; |
|
||||||
} |
|
||||||
iter_->SeekToTopFirst(); |
|
||||||
} |
|
||||||
|
|
||||||
void TruncatedRangeDelIterator::SeekToLast() { |
|
||||||
if (largest_ != nullptr) { |
|
||||||
iter_->SeekForPrev(largest_->user_key); |
|
||||||
return; |
|
||||||
} |
|
||||||
iter_->SeekToTopLast(); |
|
||||||
} |
|
||||||
|
|
||||||
std::map<SequenceNumber, std::unique_ptr<TruncatedRangeDelIterator>> |
|
||||||
TruncatedRangeDelIterator::SplitBySnapshot( |
|
||||||
const std::vector<SequenceNumber>& snapshots) { |
|
||||||
using FragmentedIterPair = |
|
||||||
std::pair<const SequenceNumber, |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator>>; |
|
||||||
|
|
||||||
auto split_untruncated_iters = iter_->SplitBySnapshot(snapshots); |
|
||||||
std::map<SequenceNumber, std::unique_ptr<TruncatedRangeDelIterator>> |
|
||||||
split_truncated_iters; |
|
||||||
std::for_each( |
|
||||||
split_untruncated_iters.begin(), split_untruncated_iters.end(), |
|
||||||
[&](FragmentedIterPair& iter_pair) { |
|
||||||
std::unique_ptr<TruncatedRangeDelIterator> truncated_iter( |
|
||||||
new TruncatedRangeDelIterator(std::move(iter_pair.second), icmp_, |
|
||||||
smallest_ikey_, largest_ikey_)); |
|
||||||
split_truncated_iters.emplace(iter_pair.first, |
|
||||||
std::move(truncated_iter)); |
|
||||||
}); |
|
||||||
return split_truncated_iters; |
|
||||||
} |
|
||||||
|
|
||||||
ForwardRangeDelIterator::ForwardRangeDelIterator( |
|
||||||
const InternalKeyComparator* icmp, |
|
||||||
const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>* iters) |
|
||||||
: icmp_(icmp), |
|
||||||
iters_(iters), |
|
||||||
unused_idx_(0), |
|
||||||
active_seqnums_(SeqMaxComparator()), |
|
||||||
active_iters_(EndKeyMinComparator(icmp)), |
|
||||||
inactive_iters_(StartKeyMinComparator(icmp)) {} |
|
||||||
|
|
||||||
bool ForwardRangeDelIterator::ShouldDelete(const ParsedInternalKey& parsed) { |
|
||||||
assert(iters_ != nullptr); |
|
||||||
// Move active iterators that end before parsed.
|
|
||||||
while (!active_iters_.empty() && |
|
||||||
icmp_->Compare((*active_iters_.top())->end_key(), parsed) <= 0) { |
|
||||||
TruncatedRangeDelIterator* iter = PopActiveIter(); |
|
||||||
do { |
|
||||||
iter->Next(); |
|
||||||
} while (iter->Valid() && icmp_->Compare(iter->end_key(), parsed) <= 0); |
|
||||||
PushIter(iter, parsed); |
|
||||||
assert(active_iters_.size() == active_seqnums_.size()); |
|
||||||
} |
|
||||||
|
|
||||||
// Move inactive iterators that start before parsed.
|
|
||||||
while (!inactive_iters_.empty() && |
|
||||||
icmp_->Compare(inactive_iters_.top()->start_key(), parsed) <= 0) { |
|
||||||
TruncatedRangeDelIterator* iter = PopInactiveIter(); |
|
||||||
while (iter->Valid() && icmp_->Compare(iter->end_key(), parsed) <= 0) { |
|
||||||
iter->Next(); |
|
||||||
} |
|
||||||
PushIter(iter, parsed); |
|
||||||
assert(active_iters_.size() == active_seqnums_.size()); |
|
||||||
} |
|
||||||
|
|
||||||
return active_seqnums_.empty() |
|
||||||
? false |
|
||||||
: (*active_seqnums_.begin())->seq() > parsed.sequence; |
|
||||||
} |
|
||||||
|
|
||||||
void ForwardRangeDelIterator::Invalidate() { |
|
||||||
unused_idx_ = 0; |
|
||||||
active_iters_.clear(); |
|
||||||
active_seqnums_.clear(); |
|
||||||
inactive_iters_.clear(); |
|
||||||
} |
|
||||||
|
|
||||||
ReverseRangeDelIterator::ReverseRangeDelIterator( |
|
||||||
const InternalKeyComparator* icmp, |
|
||||||
const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>* iters) |
|
||||||
: icmp_(icmp), |
|
||||||
iters_(iters), |
|
||||||
unused_idx_(0), |
|
||||||
active_seqnums_(SeqMaxComparator()), |
|
||||||
active_iters_(StartKeyMaxComparator(icmp)), |
|
||||||
inactive_iters_(EndKeyMaxComparator(icmp)) {} |
|
||||||
|
|
||||||
bool ReverseRangeDelIterator::ShouldDelete(const ParsedInternalKey& parsed) { |
|
||||||
assert(iters_ != nullptr); |
|
||||||
// Move active iterators that start after parsed.
|
|
||||||
while (!active_iters_.empty() && |
|
||||||
icmp_->Compare(parsed, (*active_iters_.top())->start_key()) < 0) { |
|
||||||
TruncatedRangeDelIterator* iter = PopActiveIter(); |
|
||||||
do { |
|
||||||
iter->Prev(); |
|
||||||
} while (iter->Valid() && icmp_->Compare(parsed, iter->start_key()) < 0); |
|
||||||
PushIter(iter, parsed); |
|
||||||
assert(active_iters_.size() == active_seqnums_.size()); |
|
||||||
} |
|
||||||
|
|
||||||
// Move inactive iterators that end after parsed.
|
|
||||||
while (!inactive_iters_.empty() && |
|
||||||
icmp_->Compare(parsed, inactive_iters_.top()->end_key()) < 0) { |
|
||||||
TruncatedRangeDelIterator* iter = PopInactiveIter(); |
|
||||||
while (iter->Valid() && icmp_->Compare(parsed, iter->start_key()) < 0) { |
|
||||||
iter->Prev(); |
|
||||||
} |
|
||||||
PushIter(iter, parsed); |
|
||||||
assert(active_iters_.size() == active_seqnums_.size()); |
|
||||||
} |
|
||||||
|
|
||||||
return active_seqnums_.empty() |
|
||||||
? false |
|
||||||
: (*active_seqnums_.begin())->seq() > parsed.sequence; |
|
||||||
} |
|
||||||
|
|
||||||
void ReverseRangeDelIterator::Invalidate() { |
|
||||||
unused_idx_ = 0; |
|
||||||
active_iters_.clear(); |
|
||||||
active_seqnums_.clear(); |
|
||||||
inactive_iters_.clear(); |
|
||||||
} |
|
||||||
|
|
||||||
bool RangeDelAggregatorV2::StripeRep::ShouldDelete( |
|
||||||
const ParsedInternalKey& parsed, RangeDelPositioningMode mode) { |
|
||||||
if (!InStripe(parsed.sequence) || IsEmpty()) { |
|
||||||
return false; |
|
||||||
} |
|
||||||
switch (mode) { |
|
||||||
case RangeDelPositioningMode::kForwardTraversal: |
|
||||||
InvalidateReverseIter(); |
|
||||||
|
|
||||||
// Pick up previously unseen iterators.
|
|
||||||
for (auto it = std::next(iters_.begin(), forward_iter_.UnusedIdx()); |
|
||||||
it != iters_.end(); ++it, forward_iter_.IncUnusedIdx()) { |
|
||||||
auto& iter = *it; |
|
||||||
forward_iter_.AddNewIter(iter.get(), parsed); |
|
||||||
} |
|
||||||
|
|
||||||
return forward_iter_.ShouldDelete(parsed); |
|
||||||
case RangeDelPositioningMode::kBackwardTraversal: |
|
||||||
InvalidateForwardIter(); |
|
||||||
|
|
||||||
// Pick up previously unseen iterators.
|
|
||||||
for (auto it = std::next(iters_.begin(), reverse_iter_.UnusedIdx()); |
|
||||||
it != iters_.end(); ++it, reverse_iter_.IncUnusedIdx()) { |
|
||||||
auto& iter = *it; |
|
||||||
reverse_iter_.AddNewIter(iter.get(), parsed); |
|
||||||
} |
|
||||||
|
|
||||||
return reverse_iter_.ShouldDelete(parsed); |
|
||||||
default: |
|
||||||
assert(false); |
|
||||||
return false; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
bool RangeDelAggregatorV2::StripeRep::IsRangeOverlapped(const Slice& start, |
|
||||||
const Slice& end) { |
|
||||||
Invalidate(); |
|
||||||
|
|
||||||
// Set the internal start/end keys so that:
|
|
||||||
// - if start_ikey has the same user key and sequence number as the
|
|
||||||
// current end key, start_ikey will be considered greater; and
|
|
||||||
// - if end_ikey has the same user key and sequence number as the current
|
|
||||||
// start key, end_ikey will be considered greater.
|
|
||||||
ParsedInternalKey start_ikey(start, kMaxSequenceNumber, |
|
||||||
static_cast<ValueType>(0)); |
|
||||||
ParsedInternalKey end_ikey(end, 0, static_cast<ValueType>(0)); |
|
||||||
for (auto& iter : iters_) { |
|
||||||
bool checked_candidate_tombstones = false; |
|
||||||
for (iter->SeekForPrev(start); |
|
||||||
iter->Valid() && icmp_->Compare(iter->start_key(), end_ikey) <= 0; |
|
||||||
iter->Next()) { |
|
||||||
checked_candidate_tombstones = true; |
|
||||||
if (icmp_->Compare(start_ikey, iter->end_key()) < 0 && |
|
||||||
icmp_->Compare(iter->start_key(), end_ikey) <= 0) { |
|
||||||
return true; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
if (!checked_candidate_tombstones) { |
|
||||||
// Do an additional check for when the end of the range is the begin
|
|
||||||
// key of a tombstone, which we missed earlier since SeekForPrev'ing
|
|
||||||
// to the start was invalid.
|
|
||||||
iter->SeekForPrev(end); |
|
||||||
if (iter->Valid() && icmp_->Compare(start_ikey, iter->end_key()) < 0 && |
|
||||||
icmp_->Compare(iter->start_key(), end_ikey) <= 0) { |
|
||||||
return true; |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
return false; |
|
||||||
} |
|
||||||
|
|
||||||
void ReadRangeDelAggregatorV2::AddTombstones( |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
|
||||||
const InternalKey* smallest, const InternalKey* largest) { |
|
||||||
if (input_iter == nullptr || input_iter->empty()) { |
|
||||||
return; |
|
||||||
} |
|
||||||
rep_.AddTombstones( |
|
||||||
std::unique_ptr<TruncatedRangeDelIterator>(new TruncatedRangeDelIterator( |
|
||||||
std::move(input_iter), icmp_, smallest, largest))); |
|
||||||
} |
|
||||||
|
|
||||||
bool ReadRangeDelAggregatorV2::ShouldDelete(const ParsedInternalKey& parsed, |
|
||||||
RangeDelPositioningMode mode) { |
|
||||||
return rep_.ShouldDelete(parsed, mode); |
|
||||||
} |
|
||||||
|
|
||||||
bool ReadRangeDelAggregatorV2::IsRangeOverlapped(const Slice& start, |
|
||||||
const Slice& end) { |
|
||||||
InvalidateRangeDelMapPositions(); |
|
||||||
return rep_.IsRangeOverlapped(start, end); |
|
||||||
} |
|
||||||
|
|
||||||
void CompactionRangeDelAggregatorV2::AddTombstones( |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
|
||||||
const InternalKey* smallest, const InternalKey* largest) { |
|
||||||
if (input_iter == nullptr || input_iter->empty()) { |
|
||||||
return; |
|
||||||
} |
|
||||||
assert(input_iter->lower_bound() == 0); |
|
||||||
assert(input_iter->upper_bound() == kMaxSequenceNumber); |
|
||||||
parent_iters_.emplace_back(new TruncatedRangeDelIterator( |
|
||||||
std::move(input_iter), icmp_, smallest, largest)); |
|
||||||
|
|
||||||
auto split_iters = parent_iters_.back()->SplitBySnapshot(*snapshots_); |
|
||||||
for (auto& split_iter : split_iters) { |
|
||||||
auto it = reps_.find(split_iter.first); |
|
||||||
if (it == reps_.end()) { |
|
||||||
bool inserted; |
|
||||||
SequenceNumber upper_bound = split_iter.second->upper_bound(); |
|
||||||
SequenceNumber lower_bound = split_iter.second->lower_bound(); |
|
||||||
std::tie(it, inserted) = reps_.emplace( |
|
||||||
split_iter.first, StripeRep(icmp_, upper_bound, lower_bound)); |
|
||||||
assert(inserted); |
|
||||||
} |
|
||||||
assert(it != reps_.end()); |
|
||||||
it->second.AddTombstones(std::move(split_iter.second)); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
bool CompactionRangeDelAggregatorV2::ShouldDelete( |
|
||||||
const ParsedInternalKey& parsed, RangeDelPositioningMode mode) { |
|
||||||
auto it = reps_.lower_bound(parsed.sequence); |
|
||||||
if (it == reps_.end()) { |
|
||||||
return false; |
|
||||||
} |
|
||||||
return it->second.ShouldDelete(parsed, mode); |
|
||||||
} |
|
||||||
|
|
||||||
namespace { |
|
||||||
|
|
||||||
class TruncatedRangeDelMergingIter : public InternalIterator { |
|
||||||
public: |
|
||||||
TruncatedRangeDelMergingIter( |
|
||||||
const InternalKeyComparator* icmp, const Slice* lower_bound, |
|
||||||
const Slice* upper_bound, bool upper_bound_inclusive, |
|
||||||
const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>& children) |
|
||||||
: icmp_(icmp), |
|
||||||
lower_bound_(lower_bound), |
|
||||||
upper_bound_(upper_bound), |
|
||||||
upper_bound_inclusive_(upper_bound_inclusive), |
|
||||||
heap_(StartKeyMinComparator(icmp)) { |
|
||||||
for (auto& child : children) { |
|
||||||
if (child != nullptr) { |
|
||||||
assert(child->lower_bound() == 0); |
|
||||||
assert(child->upper_bound() == kMaxSequenceNumber); |
|
||||||
children_.push_back(child.get()); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
bool Valid() const override { |
|
||||||
return !heap_.empty() && BeforeEndKey(heap_.top()); |
|
||||||
} |
|
||||||
Status status() const override { return Status::OK(); } |
|
||||||
|
|
||||||
void SeekToFirst() override { |
|
||||||
heap_.clear(); |
|
||||||
for (auto& child : children_) { |
|
||||||
if (lower_bound_ != nullptr) { |
|
||||||
child->Seek(*lower_bound_); |
|
||||||
} else { |
|
||||||
child->SeekToFirst(); |
|
||||||
} |
|
||||||
if (child->Valid()) { |
|
||||||
heap_.push(child); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
void Next() override { |
|
||||||
auto* top = heap_.top(); |
|
||||||
top->InternalNext(); |
|
||||||
if (top->Valid()) { |
|
||||||
heap_.replace_top(top); |
|
||||||
} else { |
|
||||||
heap_.pop(); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
Slice key() const override { |
|
||||||
auto* top = heap_.top(); |
|
||||||
cur_start_key_.Set(top->start_key().user_key, top->seq(), |
|
||||||
kTypeRangeDeletion); |
|
||||||
return cur_start_key_.Encode(); |
|
||||||
} |
|
||||||
|
|
||||||
Slice value() const override { |
|
||||||
auto* top = heap_.top(); |
|
||||||
assert(top->end_key().sequence == kMaxSequenceNumber); |
|
||||||
return top->end_key().user_key; |
|
||||||
} |
|
||||||
|
|
||||||
// Unused InternalIterator methods
|
|
||||||
void Prev() override { assert(false); } |
|
||||||
void Seek(const Slice& /* target */) override { assert(false); } |
|
||||||
void SeekForPrev(const Slice& /* target */) override { assert(false); } |
|
||||||
void SeekToLast() override { assert(false); } |
|
||||||
|
|
||||||
private: |
|
||||||
bool BeforeEndKey(const TruncatedRangeDelIterator* iter) const { |
|
||||||
if (upper_bound_ == nullptr) { |
|
||||||
return true; |
|
||||||
} |
|
||||||
int cmp = icmp_->user_comparator()->Compare(iter->start_key().user_key, |
|
||||||
*upper_bound_); |
|
||||||
return upper_bound_inclusive_ ? cmp <= 0 : cmp < 0; |
|
||||||
} |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp_; |
|
||||||
const Slice* lower_bound_; |
|
||||||
const Slice* upper_bound_; |
|
||||||
bool upper_bound_inclusive_; |
|
||||||
BinaryHeap<TruncatedRangeDelIterator*, StartKeyMinComparator> heap_; |
|
||||||
std::vector<TruncatedRangeDelIterator*> children_; |
|
||||||
|
|
||||||
mutable InternalKey cur_start_key_; |
|
||||||
}; |
|
||||||
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> |
|
||||||
CompactionRangeDelAggregatorV2::NewIterator(const Slice* lower_bound, |
|
||||||
const Slice* upper_bound, |
|
||||||
bool upper_bound_inclusive) { |
|
||||||
InvalidateRangeDelMapPositions(); |
|
||||||
std::unique_ptr<TruncatedRangeDelMergingIter> merging_iter( |
|
||||||
new TruncatedRangeDelMergingIter(icmp_, lower_bound, upper_bound, |
|
||||||
upper_bound_inclusive, parent_iters_)); |
|
||||||
|
|
||||||
// TODO: add tests where tombstone fragments can be outside of upper and lower
|
|
||||||
// bound range
|
|
||||||
auto fragmented_tombstone_list = |
|
||||||
std::make_shared<FragmentedRangeTombstoneList>( |
|
||||||
std::move(merging_iter), *icmp_, true /* for_compaction */, |
|
||||||
*snapshots_); |
|
||||||
|
|
||||||
return std::unique_ptr<FragmentedRangeTombstoneIterator>( |
|
||||||
new FragmentedRangeTombstoneIterator( |
|
||||||
fragmented_tombstone_list, *icmp_, |
|
||||||
kMaxSequenceNumber /* upper_bound */)); |
|
||||||
} |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
@ -1,436 +0,0 @@ |
|||||||
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
#pragma once |
|
||||||
|
|
||||||
#include <algorithm> |
|
||||||
#include <iterator> |
|
||||||
#include <list> |
|
||||||
#include <map> |
|
||||||
#include <set> |
|
||||||
#include <string> |
|
||||||
#include <vector> |
|
||||||
|
|
||||||
#include "db/compaction_iteration_stats.h" |
|
||||||
#include "db/dbformat.h" |
|
||||||
#include "db/pinned_iterators_manager.h" |
|
||||||
#include "db/range_del_aggregator.h" |
|
||||||
#include "db/range_tombstone_fragmenter.h" |
|
||||||
#include "db/version_edit.h" |
|
||||||
#include "include/rocksdb/comparator.h" |
|
||||||
#include "include/rocksdb/types.h" |
|
||||||
#include "table/internal_iterator.h" |
|
||||||
#include "table/scoped_arena_iterator.h" |
|
||||||
#include "table/table_builder.h" |
|
||||||
#include "util/heap.h" |
|
||||||
#include "util/kv_map.h" |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
class TruncatedRangeDelIterator { |
|
||||||
public: |
|
||||||
TruncatedRangeDelIterator( |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> iter, |
|
||||||
const InternalKeyComparator* icmp, const InternalKey* smallest, |
|
||||||
const InternalKey* largest); |
|
||||||
|
|
||||||
bool Valid() const; |
|
||||||
|
|
||||||
void Next(); |
|
||||||
void Prev(); |
|
||||||
|
|
||||||
void InternalNext(); |
|
||||||
|
|
||||||
// Seeks to the tombstone with the highest viisble sequence number that covers
|
|
||||||
// target (a user key). If no such tombstone exists, the position will be at
|
|
||||||
// the earliest tombstone that ends after target.
|
|
||||||
void Seek(const Slice& target); |
|
||||||
|
|
||||||
// Seeks to the tombstone with the highest viisble sequence number that covers
|
|
||||||
// target (a user key). If no such tombstone exists, the position will be at
|
|
||||||
// the latest tombstone that starts before target.
|
|
||||||
void SeekForPrev(const Slice& target); |
|
||||||
|
|
||||||
void SeekToFirst(); |
|
||||||
void SeekToLast(); |
|
||||||
|
|
||||||
ParsedInternalKey start_key() const { |
|
||||||
return (smallest_ == nullptr || |
|
||||||
icmp_->Compare(*smallest_, iter_->parsed_start_key()) <= 0) |
|
||||||
? iter_->parsed_start_key() |
|
||||||
: *smallest_; |
|
||||||
} |
|
||||||
|
|
||||||
ParsedInternalKey end_key() const { |
|
||||||
return (largest_ == nullptr || |
|
||||||
icmp_->Compare(iter_->parsed_end_key(), *largest_) <= 0) |
|
||||||
? iter_->parsed_end_key() |
|
||||||
: *largest_; |
|
||||||
} |
|
||||||
|
|
||||||
SequenceNumber seq() const { return iter_->seq(); } |
|
||||||
|
|
||||||
std::map<SequenceNumber, std::unique_ptr<TruncatedRangeDelIterator>> |
|
||||||
SplitBySnapshot(const std::vector<SequenceNumber>& snapshots); |
|
||||||
|
|
||||||
SequenceNumber upper_bound() const { return iter_->upper_bound(); } |
|
||||||
|
|
||||||
SequenceNumber lower_bound() const { return iter_->lower_bound(); } |
|
||||||
|
|
||||||
private: |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> iter_; |
|
||||||
const InternalKeyComparator* icmp_; |
|
||||||
const ParsedInternalKey* smallest_ = nullptr; |
|
||||||
const ParsedInternalKey* largest_ = nullptr; |
|
||||||
std::list<ParsedInternalKey> pinned_bounds_; |
|
||||||
|
|
||||||
const InternalKey* smallest_ikey_; |
|
||||||
const InternalKey* largest_ikey_; |
|
||||||
}; |
|
||||||
|
|
||||||
struct SeqMaxComparator { |
|
||||||
bool operator()(const TruncatedRangeDelIterator* a, |
|
||||||
const TruncatedRangeDelIterator* b) const { |
|
||||||
return a->seq() > b->seq(); |
|
||||||
} |
|
||||||
}; |
|
||||||
|
|
||||||
struct StartKeyMinComparator { |
|
||||||
explicit StartKeyMinComparator(const InternalKeyComparator* c) : icmp(c) {} |
|
||||||
|
|
||||||
bool operator()(const TruncatedRangeDelIterator* a, |
|
||||||
const TruncatedRangeDelIterator* b) const { |
|
||||||
return icmp->Compare(a->start_key(), b->start_key()) > 0; |
|
||||||
} |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp; |
|
||||||
}; |
|
||||||
|
|
||||||
class ForwardRangeDelIterator { |
|
||||||
public: |
|
||||||
ForwardRangeDelIterator( |
|
||||||
const InternalKeyComparator* icmp, |
|
||||||
const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>* iters); |
|
||||||
|
|
||||||
bool ShouldDelete(const ParsedInternalKey& parsed); |
|
||||||
void Invalidate(); |
|
||||||
|
|
||||||
void AddNewIter(TruncatedRangeDelIterator* iter, |
|
||||||
const ParsedInternalKey& parsed) { |
|
||||||
iter->Seek(parsed.user_key); |
|
||||||
PushIter(iter, parsed); |
|
||||||
assert(active_iters_.size() == active_seqnums_.size()); |
|
||||||
} |
|
||||||
|
|
||||||
size_t UnusedIdx() const { return unused_idx_; } |
|
||||||
void IncUnusedIdx() { unused_idx_++; } |
|
||||||
|
|
||||||
private: |
|
||||||
using ActiveSeqSet = |
|
||||||
std::multiset<TruncatedRangeDelIterator*, SeqMaxComparator>; |
|
||||||
|
|
||||||
struct EndKeyMinComparator { |
|
||||||
explicit EndKeyMinComparator(const InternalKeyComparator* c) : icmp(c) {} |
|
||||||
|
|
||||||
bool operator()(const ActiveSeqSet::const_iterator& a, |
|
||||||
const ActiveSeqSet::const_iterator& b) const { |
|
||||||
return icmp->Compare((*a)->end_key(), (*b)->end_key()) > 0; |
|
||||||
} |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp; |
|
||||||
}; |
|
||||||
|
|
||||||
void PushIter(TruncatedRangeDelIterator* iter, |
|
||||||
const ParsedInternalKey& parsed) { |
|
||||||
if (!iter->Valid()) { |
|
||||||
// The iterator has been fully consumed, so we don't need to add it to
|
|
||||||
// either of the heaps.
|
|
||||||
return; |
|
||||||
} |
|
||||||
int cmp = icmp_->Compare(parsed, iter->start_key()); |
|
||||||
if (cmp < 0) { |
|
||||||
PushInactiveIter(iter); |
|
||||||
} else { |
|
||||||
PushActiveIter(iter); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
void PushActiveIter(TruncatedRangeDelIterator* iter) { |
|
||||||
auto seq_pos = active_seqnums_.insert(iter); |
|
||||||
active_iters_.push(seq_pos); |
|
||||||
} |
|
||||||
|
|
||||||
TruncatedRangeDelIterator* PopActiveIter() { |
|
||||||
auto active_top = active_iters_.top(); |
|
||||||
auto iter = *active_top; |
|
||||||
active_iters_.pop(); |
|
||||||
active_seqnums_.erase(active_top); |
|
||||||
return iter; |
|
||||||
} |
|
||||||
|
|
||||||
void PushInactiveIter(TruncatedRangeDelIterator* iter) { |
|
||||||
inactive_iters_.push(iter); |
|
||||||
} |
|
||||||
|
|
||||||
TruncatedRangeDelIterator* PopInactiveIter() { |
|
||||||
auto* iter = inactive_iters_.top(); |
|
||||||
inactive_iters_.pop(); |
|
||||||
return iter; |
|
||||||
} |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp_; |
|
||||||
const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>* iters_; |
|
||||||
size_t unused_idx_; |
|
||||||
ActiveSeqSet active_seqnums_; |
|
||||||
BinaryHeap<ActiveSeqSet::const_iterator, EndKeyMinComparator> active_iters_; |
|
||||||
BinaryHeap<TruncatedRangeDelIterator*, StartKeyMinComparator> inactive_iters_; |
|
||||||
}; |
|
||||||
|
|
||||||
class ReverseRangeDelIterator { |
|
||||||
public: |
|
||||||
ReverseRangeDelIterator( |
|
||||||
const InternalKeyComparator* icmp, |
|
||||||
const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>* iters); |
|
||||||
|
|
||||||
bool ShouldDelete(const ParsedInternalKey& parsed); |
|
||||||
void Invalidate(); |
|
||||||
|
|
||||||
void AddNewIter(TruncatedRangeDelIterator* iter, |
|
||||||
const ParsedInternalKey& parsed) { |
|
||||||
iter->SeekForPrev(parsed.user_key); |
|
||||||
PushIter(iter, parsed); |
|
||||||
assert(active_iters_.size() == active_seqnums_.size()); |
|
||||||
} |
|
||||||
|
|
||||||
size_t UnusedIdx() const { return unused_idx_; } |
|
||||||
void IncUnusedIdx() { unused_idx_++; } |
|
||||||
|
|
||||||
private: |
|
||||||
using ActiveSeqSet = |
|
||||||
std::multiset<TruncatedRangeDelIterator*, SeqMaxComparator>; |
|
||||||
|
|
||||||
struct EndKeyMaxComparator { |
|
||||||
explicit EndKeyMaxComparator(const InternalKeyComparator* c) : icmp(c) {} |
|
||||||
|
|
||||||
bool operator()(const TruncatedRangeDelIterator* a, |
|
||||||
const TruncatedRangeDelIterator* b) const { |
|
||||||
return icmp->Compare(a->end_key(), b->end_key()) < 0; |
|
||||||
} |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp; |
|
||||||
}; |
|
||||||
struct StartKeyMaxComparator { |
|
||||||
explicit StartKeyMaxComparator(const InternalKeyComparator* c) : icmp(c) {} |
|
||||||
|
|
||||||
bool operator()(const ActiveSeqSet::const_iterator& a, |
|
||||||
const ActiveSeqSet::const_iterator& b) const { |
|
||||||
return icmp->Compare((*a)->start_key(), (*b)->start_key()) < 0; |
|
||||||
} |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp; |
|
||||||
}; |
|
||||||
|
|
||||||
void PushIter(TruncatedRangeDelIterator* iter, |
|
||||||
const ParsedInternalKey& parsed) { |
|
||||||
if (!iter->Valid()) { |
|
||||||
// The iterator has been fully consumed, so we don't need to add it to
|
|
||||||
// either of the heaps.
|
|
||||||
} else if (icmp_->Compare(iter->end_key(), parsed) <= 0) { |
|
||||||
PushInactiveIter(iter); |
|
||||||
} else { |
|
||||||
PushActiveIter(iter); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
void PushActiveIter(TruncatedRangeDelIterator* iter) { |
|
||||||
auto seq_pos = active_seqnums_.insert(iter); |
|
||||||
active_iters_.push(seq_pos); |
|
||||||
} |
|
||||||
|
|
||||||
TruncatedRangeDelIterator* PopActiveIter() { |
|
||||||
auto active_top = active_iters_.top(); |
|
||||||
auto iter = *active_top; |
|
||||||
active_iters_.pop(); |
|
||||||
active_seqnums_.erase(active_top); |
|
||||||
return iter; |
|
||||||
} |
|
||||||
|
|
||||||
void PushInactiveIter(TruncatedRangeDelIterator* iter) { |
|
||||||
inactive_iters_.push(iter); |
|
||||||
} |
|
||||||
|
|
||||||
TruncatedRangeDelIterator* PopInactiveIter() { |
|
||||||
auto* iter = inactive_iters_.top(); |
|
||||||
inactive_iters_.pop(); |
|
||||||
return iter; |
|
||||||
} |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp_; |
|
||||||
const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>* iters_; |
|
||||||
size_t unused_idx_; |
|
||||||
ActiveSeqSet active_seqnums_; |
|
||||||
BinaryHeap<ActiveSeqSet::const_iterator, StartKeyMaxComparator> active_iters_; |
|
||||||
BinaryHeap<TruncatedRangeDelIterator*, EndKeyMaxComparator> inactive_iters_; |
|
||||||
}; |
|
||||||
|
|
||||||
class RangeDelAggregatorV2 { |
|
||||||
public: |
|
||||||
explicit RangeDelAggregatorV2(const InternalKeyComparator* icmp) |
|
||||||
: icmp_(icmp) {} |
|
||||||
virtual ~RangeDelAggregatorV2() {} |
|
||||||
|
|
||||||
virtual void AddTombstones( |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
|
||||||
const InternalKey* smallest = nullptr, |
|
||||||
const InternalKey* largest = nullptr) = 0; |
|
||||||
|
|
||||||
bool ShouldDelete(const Slice& key, RangeDelPositioningMode mode) { |
|
||||||
ParsedInternalKey parsed; |
|
||||||
if (!ParseInternalKey(key, &parsed)) { |
|
||||||
return false; |
|
||||||
} |
|
||||||
return ShouldDelete(parsed, mode); |
|
||||||
} |
|
||||||
virtual bool ShouldDelete(const ParsedInternalKey& parsed, |
|
||||||
RangeDelPositioningMode mode) = 0; |
|
||||||
|
|
||||||
virtual void InvalidateRangeDelMapPositions() = 0; |
|
||||||
|
|
||||||
virtual bool IsEmpty() const = 0; |
|
||||||
|
|
||||||
bool AddFile(uint64_t file_number) { |
|
||||||
return files_seen_.insert(file_number).second; |
|
||||||
} |
|
||||||
|
|
||||||
protected: |
|
||||||
class StripeRep { |
|
||||||
public: |
|
||||||
StripeRep(const InternalKeyComparator* icmp, SequenceNumber upper_bound, |
|
||||||
SequenceNumber lower_bound) |
|
||||||
: icmp_(icmp), |
|
||||||
forward_iter_(icmp, &iters_), |
|
||||||
reverse_iter_(icmp, &iters_), |
|
||||||
upper_bound_(upper_bound), |
|
||||||
lower_bound_(lower_bound) {} |
|
||||||
|
|
||||||
void AddTombstones(std::unique_ptr<TruncatedRangeDelIterator> input_iter) { |
|
||||||
iters_.push_back(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
bool IsEmpty() const { return iters_.empty(); } |
|
||||||
|
|
||||||
bool ShouldDelete(const ParsedInternalKey& parsed, |
|
||||||
RangeDelPositioningMode mode); |
|
||||||
|
|
||||||
void Invalidate() { |
|
||||||
InvalidateForwardIter(); |
|
||||||
InvalidateReverseIter(); |
|
||||||
} |
|
||||||
|
|
||||||
bool IsRangeOverlapped(const Slice& start, const Slice& end); |
|
||||||
|
|
||||||
private: |
|
||||||
bool InStripe(SequenceNumber seq) const { |
|
||||||
return lower_bound_ <= seq && seq <= upper_bound_; |
|
||||||
} |
|
||||||
|
|
||||||
void InvalidateForwardIter() { forward_iter_.Invalidate(); } |
|
||||||
|
|
||||||
void InvalidateReverseIter() { reverse_iter_.Invalidate(); } |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp_; |
|
||||||
std::vector<std::unique_ptr<TruncatedRangeDelIterator>> iters_; |
|
||||||
ForwardRangeDelIterator forward_iter_; |
|
||||||
ReverseRangeDelIterator reverse_iter_; |
|
||||||
SequenceNumber upper_bound_; |
|
||||||
SequenceNumber lower_bound_; |
|
||||||
}; |
|
||||||
|
|
||||||
const InternalKeyComparator* icmp_; |
|
||||||
|
|
||||||
private: |
|
||||||
std::set<uint64_t> files_seen_; |
|
||||||
}; |
|
||||||
|
|
||||||
class ReadRangeDelAggregatorV2 : public RangeDelAggregatorV2 { |
|
||||||
public: |
|
||||||
ReadRangeDelAggregatorV2(const InternalKeyComparator* icmp, |
|
||||||
SequenceNumber upper_bound) |
|
||||||
: RangeDelAggregatorV2(icmp), |
|
||||||
rep_(icmp, upper_bound, 0 /* lower_bound */) {} |
|
||||||
~ReadRangeDelAggregatorV2() override {} |
|
||||||
|
|
||||||
using RangeDelAggregatorV2::ShouldDelete; |
|
||||||
void AddTombstones( |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
|
||||||
const InternalKey* smallest = nullptr, |
|
||||||
const InternalKey* largest = nullptr) override; |
|
||||||
|
|
||||||
bool ShouldDelete(const ParsedInternalKey& parsed, |
|
||||||
RangeDelPositioningMode mode) override; |
|
||||||
|
|
||||||
bool IsRangeOverlapped(const Slice& start, const Slice& end); |
|
||||||
|
|
||||||
void InvalidateRangeDelMapPositions() override { rep_.Invalidate(); } |
|
||||||
|
|
||||||
bool IsEmpty() const override { return rep_.IsEmpty(); } |
|
||||||
|
|
||||||
private: |
|
||||||
StripeRep rep_; |
|
||||||
}; |
|
||||||
|
|
||||||
class CompactionRangeDelAggregatorV2 : public RangeDelAggregatorV2 { |
|
||||||
public: |
|
||||||
CompactionRangeDelAggregatorV2(const InternalKeyComparator* icmp, |
|
||||||
const std::vector<SequenceNumber>& snapshots) |
|
||||||
: RangeDelAggregatorV2(icmp), snapshots_(&snapshots) {} |
|
||||||
~CompactionRangeDelAggregatorV2() override {} |
|
||||||
|
|
||||||
void AddTombstones( |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
|
||||||
const InternalKey* smallest = nullptr, |
|
||||||
const InternalKey* largest = nullptr) override; |
|
||||||
|
|
||||||
using RangeDelAggregatorV2::ShouldDelete; |
|
||||||
bool ShouldDelete(const ParsedInternalKey& parsed, |
|
||||||
RangeDelPositioningMode mode) override; |
|
||||||
|
|
||||||
bool IsRangeOverlapped(const Slice& start, const Slice& end); |
|
||||||
|
|
||||||
void InvalidateRangeDelMapPositions() override { |
|
||||||
for (auto& rep : reps_) { |
|
||||||
rep.second.Invalidate(); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
bool IsEmpty() const override { |
|
||||||
for (const auto& rep : reps_) { |
|
||||||
if (!rep.second.IsEmpty()) { |
|
||||||
return false; |
|
||||||
} |
|
||||||
} |
|
||||||
return true; |
|
||||||
} |
|
||||||
|
|
||||||
// Creates an iterator over all the range tombstones in the aggregator, for
|
|
||||||
// use in compaction. Nullptr arguments indicate that the iterator range is
|
|
||||||
// unbounded.
|
|
||||||
// NOTE: the boundaries are used for optimization purposes to reduce the
|
|
||||||
// number of tombstones that are passed to the fragmenter; they do not
|
|
||||||
// guarantee that the resulting iterator only contains range tombstones that
|
|
||||||
// cover keys in the provided range. If required, these bounds must be
|
|
||||||
// enforced during iteration.
|
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> NewIterator( |
|
||||||
const Slice* lower_bound = nullptr, const Slice* upper_bound = nullptr, |
|
||||||
bool upper_bound_inclusive = false); |
|
||||||
|
|
||||||
private: |
|
||||||
std::vector<std::unique_ptr<TruncatedRangeDelIterator>> parent_iters_; |
|
||||||
std::map<SequenceNumber, StripeRep> reps_; |
|
||||||
|
|
||||||
const std::vector<SequenceNumber>* snapshots_; |
|
||||||
}; |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
@ -1,709 +0,0 @@ |
|||||||
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
#include "db/range_del_aggregator_v2.h" |
|
||||||
|
|
||||||
#include <memory> |
|
||||||
#include <string> |
|
||||||
#include <vector> |
|
||||||
|
|
||||||
#include "db/db_test_util.h" |
|
||||||
#include "db/dbformat.h" |
|
||||||
#include "db/range_tombstone_fragmenter.h" |
|
||||||
#include "util/testutil.h" |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
class RangeDelAggregatorV2Test : public testing::Test {}; |
|
||||||
|
|
||||||
namespace { |
|
||||||
|
|
||||||
static auto bytewise_icmp = InternalKeyComparator(BytewiseComparator()); |
|
||||||
|
|
||||||
std::unique_ptr<InternalIterator> MakeRangeDelIter( |
|
||||||
const std::vector<RangeTombstone>& range_dels) { |
|
||||||
std::vector<std::string> keys, values; |
|
||||||
for (const auto& range_del : range_dels) { |
|
||||||
auto key_and_value = range_del.Serialize(); |
|
||||||
keys.push_back(key_and_value.first.Encode().ToString()); |
|
||||||
values.push_back(key_and_value.second.ToString()); |
|
||||||
} |
|
||||||
return std::unique_ptr<test::VectorIterator>( |
|
||||||
new test::VectorIterator(keys, values)); |
|
||||||
} |
|
||||||
|
|
||||||
std::vector<std::unique_ptr<FragmentedRangeTombstoneList>> |
|
||||||
MakeFragmentedTombstoneLists( |
|
||||||
const std::vector<std::vector<RangeTombstone>>& range_dels_list) { |
|
||||||
std::vector<std::unique_ptr<FragmentedRangeTombstoneList>> fragment_lists; |
|
||||||
for (const auto& range_dels : range_dels_list) { |
|
||||||
auto range_del_iter = MakeRangeDelIter(range_dels); |
|
||||||
fragment_lists.emplace_back(new FragmentedRangeTombstoneList( |
|
||||||
std::move(range_del_iter), bytewise_icmp)); |
|
||||||
} |
|
||||||
return fragment_lists; |
|
||||||
} |
|
||||||
|
|
||||||
struct TruncatedIterScanTestCase { |
|
||||||
ParsedInternalKey start; |
|
||||||
ParsedInternalKey end; |
|
||||||
SequenceNumber seq; |
|
||||||
}; |
|
||||||
|
|
||||||
struct TruncatedIterSeekTestCase { |
|
||||||
Slice target; |
|
||||||
ParsedInternalKey start; |
|
||||||
ParsedInternalKey end; |
|
||||||
SequenceNumber seq; |
|
||||||
bool invalid; |
|
||||||
}; |
|
||||||
|
|
||||||
struct ShouldDeleteTestCase { |
|
||||||
ParsedInternalKey lookup_key; |
|
||||||
bool result; |
|
||||||
}; |
|
||||||
|
|
||||||
struct IsRangeOverlappedTestCase { |
|
||||||
Slice start; |
|
||||||
Slice end; |
|
||||||
bool result; |
|
||||||
}; |
|
||||||
|
|
||||||
ParsedInternalKey UncutEndpoint(const Slice& s) { |
|
||||||
return ParsedInternalKey(s, kMaxSequenceNumber, kTypeRangeDeletion); |
|
||||||
} |
|
||||||
|
|
||||||
ParsedInternalKey InternalValue(const Slice& key, SequenceNumber seq) { |
|
||||||
return ParsedInternalKey(key, seq, kTypeValue); |
|
||||||
} |
|
||||||
|
|
||||||
void VerifyIterator( |
|
||||||
TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, |
|
||||||
const std::vector<TruncatedIterScanTestCase>& expected_range_dels) { |
|
||||||
// Test forward iteration.
|
|
||||||
iter->SeekToFirst(); |
|
||||||
for (size_t i = 0; i < expected_range_dels.size(); i++, iter->Next()) { |
|
||||||
ASSERT_TRUE(iter->Valid()); |
|
||||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), expected_range_dels[i].start)); |
|
||||||
EXPECT_EQ(0, icmp.Compare(iter->end_key(), expected_range_dels[i].end)); |
|
||||||
EXPECT_EQ(expected_range_dels[i].seq, iter->seq()); |
|
||||||
} |
|
||||||
EXPECT_FALSE(iter->Valid()); |
|
||||||
|
|
||||||
// Test reverse iteration.
|
|
||||||
iter->SeekToLast(); |
|
||||||
std::vector<TruncatedIterScanTestCase> reverse_expected_range_dels( |
|
||||||
expected_range_dels.rbegin(), expected_range_dels.rend()); |
|
||||||
for (size_t i = 0; i < reverse_expected_range_dels.size(); |
|
||||||
i++, iter->Prev()) { |
|
||||||
ASSERT_TRUE(iter->Valid()); |
|
||||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), |
|
||||||
reverse_expected_range_dels[i].start)); |
|
||||||
EXPECT_EQ( |
|
||||||
0, icmp.Compare(iter->end_key(), reverse_expected_range_dels[i].end)); |
|
||||||
EXPECT_EQ(reverse_expected_range_dels[i].seq, iter->seq()); |
|
||||||
} |
|
||||||
EXPECT_FALSE(iter->Valid()); |
|
||||||
} |
|
||||||
|
|
||||||
void VerifySeek(TruncatedRangeDelIterator* iter, |
|
||||||
const InternalKeyComparator& icmp, |
|
||||||
const std::vector<TruncatedIterSeekTestCase>& test_cases) { |
|
||||||
for (const auto& test_case : test_cases) { |
|
||||||
iter->Seek(test_case.target); |
|
||||||
if (test_case.invalid) { |
|
||||||
ASSERT_FALSE(iter->Valid()); |
|
||||||
} else { |
|
||||||
ASSERT_TRUE(iter->Valid()); |
|
||||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); |
|
||||||
EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); |
|
||||||
EXPECT_EQ(test_case.seq, iter->seq()); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
void VerifySeekForPrev( |
|
||||||
TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, |
|
||||||
const std::vector<TruncatedIterSeekTestCase>& test_cases) { |
|
||||||
for (const auto& test_case : test_cases) { |
|
||||||
iter->SeekForPrev(test_case.target); |
|
||||||
if (test_case.invalid) { |
|
||||||
ASSERT_FALSE(iter->Valid()); |
|
||||||
} else { |
|
||||||
ASSERT_TRUE(iter->Valid()); |
|
||||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); |
|
||||||
EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); |
|
||||||
EXPECT_EQ(test_case.seq, iter->seq()); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
void VerifyShouldDelete(RangeDelAggregatorV2* range_del_agg, |
|
||||||
const std::vector<ShouldDeleteTestCase>& test_cases) { |
|
||||||
for (const auto& test_case : test_cases) { |
|
||||||
EXPECT_EQ( |
|
||||||
test_case.result, |
|
||||||
range_del_agg->ShouldDelete( |
|
||||||
test_case.lookup_key, RangeDelPositioningMode::kForwardTraversal)); |
|
||||||
} |
|
||||||
for (auto it = test_cases.rbegin(); it != test_cases.rend(); ++it) { |
|
||||||
const auto& test_case = *it; |
|
||||||
EXPECT_EQ( |
|
||||||
test_case.result, |
|
||||||
range_del_agg->ShouldDelete( |
|
||||||
test_case.lookup_key, RangeDelPositioningMode::kBackwardTraversal)); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
void VerifyIsRangeOverlapped( |
|
||||||
ReadRangeDelAggregatorV2* range_del_agg, |
|
||||||
const std::vector<IsRangeOverlappedTestCase>& test_cases) { |
|
||||||
for (const auto& test_case : test_cases) { |
|
||||||
EXPECT_EQ(test_case.result, |
|
||||||
range_del_agg->IsRangeOverlapped(test_case.start, test_case.end)); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
void CheckIterPosition(const RangeTombstone& tombstone, |
|
||||||
const FragmentedRangeTombstoneIterator* iter) { |
|
||||||
// Test InternalIterator interface.
|
|
||||||
EXPECT_EQ(tombstone.start_key_, ExtractUserKey(iter->key())); |
|
||||||
EXPECT_EQ(tombstone.end_key_, iter->value()); |
|
||||||
EXPECT_EQ(tombstone.seq_, iter->seq()); |
|
||||||
|
|
||||||
// Test FragmentedRangeTombstoneIterator interface.
|
|
||||||
EXPECT_EQ(tombstone.start_key_, iter->start_key()); |
|
||||||
EXPECT_EQ(tombstone.end_key_, iter->end_key()); |
|
||||||
EXPECT_EQ(tombstone.seq_, GetInternalKeySeqno(iter->key())); |
|
||||||
} |
|
||||||
|
|
||||||
void VerifyFragmentedRangeDels( |
|
||||||
FragmentedRangeTombstoneIterator* iter, |
|
||||||
const std::vector<RangeTombstone>& expected_tombstones) { |
|
||||||
iter->SeekToFirst(); |
|
||||||
for (size_t i = 0; i < expected_tombstones.size(); i++, iter->Next()) { |
|
||||||
ASSERT_TRUE(iter->Valid()); |
|
||||||
CheckIterPosition(expected_tombstones[i], iter); |
|
||||||
} |
|
||||||
EXPECT_FALSE(iter->Valid()); |
|
||||||
} |
|
||||||
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, EmptyTruncatedIter) { |
|
||||||
auto range_del_iter = MakeRangeDelIter({}); |
|
||||||
FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), |
|
||||||
bytewise_icmp); |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
|
|
||||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
|
||||||
nullptr); |
|
||||||
|
|
||||||
iter.SeekToFirst(); |
|
||||||
ASSERT_FALSE(iter.Valid()); |
|
||||||
|
|
||||||
iter.SeekToLast(); |
|
||||||
ASSERT_FALSE(iter.Valid()); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, UntruncatedIter) { |
|
||||||
auto range_del_iter = |
|
||||||
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
|
||||||
FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), |
|
||||||
bytewise_icmp); |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
|
|
||||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
|
||||||
nullptr); |
|
||||||
|
|
||||||
VerifyIterator(&iter, bytewise_icmp, |
|
||||||
{{UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
|
||||||
{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{UncutEndpoint("j"), UncutEndpoint("n"), 4}}); |
|
||||||
|
|
||||||
VerifySeek( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
|
||||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"ia", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
|
||||||
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
|
||||||
{"", UncutEndpoint("a"), UncutEndpoint("e"), 10}}); |
|
||||||
|
|
||||||
VerifySeekForPrev( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
|
||||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"n", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
|
||||||
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, UntruncatedIterWithSnapshot) { |
|
||||||
auto range_del_iter = |
|
||||||
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
|
||||||
FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), |
|
||||||
bytewise_icmp); |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, |
|
||||||
9 /* snapshot */)); |
|
||||||
|
|
||||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
|
||||||
nullptr); |
|
||||||
|
|
||||||
VerifyIterator(&iter, bytewise_icmp, |
|
||||||
{{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{UncutEndpoint("j"), UncutEndpoint("n"), 4}}); |
|
||||||
|
|
||||||
VerifySeek( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"ia", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
|
||||||
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
|
||||||
{"", UncutEndpoint("e"), UncutEndpoint("g"), 8}}); |
|
||||||
|
|
||||||
VerifySeekForPrev( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
|
||||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"n", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
|
||||||
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, TruncatedIterPartiallyCutTombstones) { |
|
||||||
auto range_del_iter = |
|
||||||
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
|
||||||
FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), |
|
||||||
bytewise_icmp); |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
|
|
||||||
InternalKey smallest("d", 7, kTypeValue); |
|
||||||
InternalKey largest("m", 9, kTypeValue); |
|
||||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, |
|
||||||
&smallest, &largest); |
|
||||||
|
|
||||||
VerifyIterator(&iter, bytewise_icmp, |
|
||||||
{{InternalValue("d", 7), UncutEndpoint("e"), 10}, |
|
||||||
{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{UncutEndpoint("j"), InternalValue("m", 8), 4}}); |
|
||||||
|
|
||||||
VerifySeek( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10}, |
|
||||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"ia", UncutEndpoint("j"), InternalValue("m", 8), 4}, |
|
||||||
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
|
||||||
{"", InternalValue("d", 7), UncutEndpoint("e"), 10}}); |
|
||||||
|
|
||||||
VerifySeekForPrev( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10}, |
|
||||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
|
||||||
{"n", UncutEndpoint("j"), InternalValue("m", 8), 4}, |
|
||||||
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, TruncatedIterFullyCutTombstones) { |
|
||||||
auto range_del_iter = |
|
||||||
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
|
||||||
FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), |
|
||||||
bytewise_icmp); |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
|
|
||||||
InternalKey smallest("f", 7, kTypeValue); |
|
||||||
InternalKey largest("i", 9, kTypeValue); |
|
||||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, |
|
||||||
&smallest, &largest); |
|
||||||
|
|
||||||
VerifyIterator(&iter, bytewise_icmp, |
|
||||||
{{InternalValue("f", 7), UncutEndpoint("g"), 8}}); |
|
||||||
|
|
||||||
VerifySeek( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", InternalValue("f", 7), UncutEndpoint("g"), 8}, |
|
||||||
{"f", InternalValue("f", 7), UncutEndpoint("g"), 8}, |
|
||||||
{"j", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
|
||||||
|
|
||||||
VerifySeekForPrev( |
|
||||||
&iter, bytewise_icmp, |
|
||||||
{{"d", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
|
||||||
{"f", InternalValue("f", 7), UncutEndpoint("g"), 8}, |
|
||||||
{"j", InternalValue("f", 7), UncutEndpoint("g"), 8}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, SingleIterInAggregator) { |
|
||||||
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, {"c", "g", 8}}); |
|
||||||
FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), |
|
||||||
bytewise_icmp); |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
|
|
||||||
ReadRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, kMaxSequenceNumber); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
|
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, |
|
||||||
{InternalValue("b", 9), true}, |
|
||||||
{InternalValue("d", 9), true}, |
|
||||||
{InternalValue("e", 7), true}, |
|
||||||
{InternalValue("g", 7), false}}); |
|
||||||
|
|
||||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
|
||||||
{"_", "a", true}, |
|
||||||
{"a", "c", true}, |
|
||||||
{"d", "f", true}, |
|
||||||
{"g", "l", false}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, MultipleItersInAggregator) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
|
||||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
ReadRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, kMaxSequenceNumber); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), true}, |
|
||||||
{InternalValue("b", 19), false}, |
|
||||||
{InternalValue("b", 9), true}, |
|
||||||
{InternalValue("d", 9), true}, |
|
||||||
{InternalValue("e", 7), true}, |
|
||||||
{InternalValue("g", 7), false}, |
|
||||||
{InternalValue("h", 24), true}, |
|
||||||
{InternalValue("i", 24), false}, |
|
||||||
{InternalValue("ii", 14), true}, |
|
||||||
{InternalValue("j", 14), false}}); |
|
||||||
|
|
||||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
|
||||||
{"_", "a", true}, |
|
||||||
{"a", "c", true}, |
|
||||||
{"d", "f", true}, |
|
||||||
{"g", "l", true}, |
|
||||||
{"x", "y", false}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, MultipleItersInAggregatorWithUpperBound) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
|
||||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
ReadRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
19 /* snapshot */)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, |
|
||||||
{InternalValue("a", 9), true}, |
|
||||||
{InternalValue("b", 9), true}, |
|
||||||
{InternalValue("d", 9), true}, |
|
||||||
{InternalValue("e", 7), true}, |
|
||||||
{InternalValue("g", 7), false}, |
|
||||||
{InternalValue("h", 24), false}, |
|
||||||
{InternalValue("i", 24), false}, |
|
||||||
{InternalValue("ii", 14), true}, |
|
||||||
{InternalValue("j", 14), false}}); |
|
||||||
|
|
||||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
|
||||||
{"_", "a", true}, |
|
||||||
{"a", "c", true}, |
|
||||||
{"d", "f", true}, |
|
||||||
{"g", "l", true}, |
|
||||||
{"x", "y", false}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, MultipleTruncatedItersInAggregator) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); |
|
||||||
std::vector<std::pair<InternalKey, InternalKey>> iter_bounds = { |
|
||||||
{InternalKey("a", 4, kTypeValue), |
|
||||||
InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, |
|
||||||
{InternalKey("m", 20, kTypeValue), |
|
||||||
InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, |
|
||||||
{InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; |
|
||||||
|
|
||||||
ReadRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
|
||||||
for (size_t i = 0; i < fragment_lists.size(); i++) { |
|
||||||
const auto& fragment_list = fragment_lists[i]; |
|
||||||
const auto& bounds = iter_bounds[i]; |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
19 /* snapshot */)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter), &bounds.first, |
|
||||||
&bounds.second); |
|
||||||
} |
|
||||||
|
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, |
|
||||||
{InternalValue("a", 9), false}, |
|
||||||
{InternalValue("a", 4), true}, |
|
||||||
{InternalValue("m", 10), false}, |
|
||||||
{InternalValue("m", 9), true}, |
|
||||||
{InternalValue("x", 10), false}, |
|
||||||
{InternalValue("x", 9), false}, |
|
||||||
{InternalValue("x", 5), true}, |
|
||||||
{InternalValue("z", 9), false}}); |
|
||||||
|
|
||||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
|
||||||
{"_", "a", true}, |
|
||||||
{"a", "n", true}, |
|
||||||
{"l", "x", true}, |
|
||||||
{"w", "z", true}, |
|
||||||
{"zzz", "zz", false}, |
|
||||||
{"zz", "zzz", false}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, MultipleTruncatedItersInAggregatorSameLevel) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); |
|
||||||
std::vector<std::pair<InternalKey, InternalKey>> iter_bounds = { |
|
||||||
{InternalKey("a", 4, kTypeValue), |
|
||||||
InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, |
|
||||||
{InternalKey("m", 20, kTypeValue), |
|
||||||
InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, |
|
||||||
{InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; |
|
||||||
|
|
||||||
ReadRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
|
||||||
|
|
||||||
auto add_iter_to_agg = [&](size_t i) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_lists[i].get(), |
|
||||||
bytewise_icmp, 19 /* snapshot */)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter), &iter_bounds[i].first, |
|
||||||
&iter_bounds[i].second); |
|
||||||
}; |
|
||||||
|
|
||||||
add_iter_to_agg(0); |
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, |
|
||||||
{InternalValue("a", 9), false}, |
|
||||||
{InternalValue("a", 4), true}}); |
|
||||||
|
|
||||||
add_iter_to_agg(1); |
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("m", 10), false}, |
|
||||||
{InternalValue("m", 9), true}}); |
|
||||||
|
|
||||||
add_iter_to_agg(2); |
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("x", 10), false}, |
|
||||||
{InternalValue("x", 9), false}, |
|
||||||
{InternalValue("x", 5), true}, |
|
||||||
{InternalValue("z", 9), false}}); |
|
||||||
|
|
||||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
|
||||||
{"_", "a", true}, |
|
||||||
{"a", "n", true}, |
|
||||||
{"l", "x", true}, |
|
||||||
{"w", "z", true}, |
|
||||||
{"zzz", "zz", false}, |
|
||||||
{"zz", "zzz", false}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, CompactionAggregatorNoSnapshots) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
|
||||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
std::vector<SequenceNumber> snapshots; |
|
||||||
CompactionRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, snapshots); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), true}, |
|
||||||
{InternalValue("b", 19), false}, |
|
||||||
{InternalValue("b", 9), true}, |
|
||||||
{InternalValue("d", 9), true}, |
|
||||||
{InternalValue("e", 7), true}, |
|
||||||
{InternalValue("g", 7), false}, |
|
||||||
{InternalValue("h", 24), true}, |
|
||||||
{InternalValue("i", 24), false}, |
|
||||||
{InternalValue("ii", 14), true}, |
|
||||||
{InternalValue("j", 14), false}}); |
|
||||||
|
|
||||||
auto range_del_compaction_iter = range_del_agg.NewIterator(); |
|
||||||
VerifyFragmentedRangeDels(range_del_compaction_iter.get(), {{"a", "b", 20}, |
|
||||||
{"b", "c", 10}, |
|
||||||
{"c", "e", 10}, |
|
||||||
{"e", "g", 8}, |
|
||||||
{"h", "i", 25}, |
|
||||||
{"ii", "j", 15}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, CompactionAggregatorWithSnapshots) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
|
||||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
std::vector<SequenceNumber> snapshots{9, 19}; |
|
||||||
CompactionRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, snapshots); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
VerifyShouldDelete( |
|
||||||
&range_del_agg, |
|
||||||
{ |
|
||||||
{InternalValue("a", 19), false}, // [10, 19]
|
|
||||||
{InternalValue("a", 9), false}, // [0, 9]
|
|
||||||
{InternalValue("b", 9), false}, // [0, 9]
|
|
||||||
{InternalValue("d", 9), false}, // [0, 9]
|
|
||||||
{InternalValue("d", 7), true}, // [0, 9]
|
|
||||||
{InternalValue("e", 7), true}, // [0, 9]
|
|
||||||
{InternalValue("g", 7), false}, // [0, 9]
|
|
||||||
{InternalValue("h", 24), true}, // [20, kMaxSequenceNumber]
|
|
||||||
{InternalValue("i", 24), false}, // [20, kMaxSequenceNumber]
|
|
||||||
{InternalValue("ii", 14), true}, // [10, 19]
|
|
||||||
{InternalValue("j", 14), false} // [10, 19]
|
|
||||||
}); |
|
||||||
|
|
||||||
auto range_del_compaction_iter = range_del_agg.NewIterator(); |
|
||||||
VerifyFragmentedRangeDels(range_del_compaction_iter.get(), {{"a", "b", 20}, |
|
||||||
{"a", "b", 10}, |
|
||||||
{"b", "c", 10}, |
|
||||||
{"c", "e", 10}, |
|
||||||
{"c", "e", 8}, |
|
||||||
{"e", "g", 8}, |
|
||||||
{"h", "i", 25}, |
|
||||||
{"ii", "j", 15}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, CompactionAggregatorEmptyIteratorLeft) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
|
||||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
std::vector<SequenceNumber> snapshots{9, 19}; |
|
||||||
CompactionRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, snapshots); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
Slice start("_"); |
|
||||||
Slice end("__"); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, CompactionAggregatorEmptyIteratorRight) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
|
||||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
std::vector<SequenceNumber> snapshots{9, 19}; |
|
||||||
CompactionRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, snapshots); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
Slice start("p"); |
|
||||||
Slice end("q"); |
|
||||||
auto range_del_compaction_iter1 = |
|
||||||
range_del_agg.NewIterator(&start, &end, false /* end_key_inclusive */); |
|
||||||
VerifyFragmentedRangeDels(range_del_compaction_iter1.get(), {}); |
|
||||||
|
|
||||||
auto range_del_compaction_iter2 = |
|
||||||
range_del_agg.NewIterator(&start, &end, true /* end_key_inclusive */); |
|
||||||
VerifyFragmentedRangeDels(range_del_compaction_iter2.get(), {}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, CompactionAggregatorBoundedIterator) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
|
||||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
std::vector<SequenceNumber> snapshots{9, 19}; |
|
||||||
CompactionRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, snapshots); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
Slice start("bb"); |
|
||||||
Slice end("e"); |
|
||||||
auto range_del_compaction_iter1 = |
|
||||||
range_del_agg.NewIterator(&start, &end, false /* end_key_inclusive */); |
|
||||||
VerifyFragmentedRangeDels(range_del_compaction_iter1.get(), |
|
||||||
{{"a", "c", 10}, {"c", "e", 10}, {"c", "e", 8}}); |
|
||||||
|
|
||||||
auto range_del_compaction_iter2 = |
|
||||||
range_del_agg.NewIterator(&start, &end, true /* end_key_inclusive */); |
|
||||||
VerifyFragmentedRangeDels( |
|
||||||
range_del_compaction_iter2.get(), |
|
||||||
{{"a", "c", 10}, {"c", "e", 10}, {"c", "e", 8}, {"e", "g", 8}}); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(RangeDelAggregatorV2Test, |
|
||||||
CompactionAggregatorBoundedIteratorExtraFragments) { |
|
||||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
|
||||||
{{{"a", "d", 10}, {"c", "g", 8}}, |
|
||||||
{{"b", "c", 20}, {"d", "f", 30}, {"h", "i", 25}, {"ii", "j", 15}}}); |
|
||||||
|
|
||||||
std::vector<SequenceNumber> snapshots{9, 19}; |
|
||||||
CompactionRangeDelAggregatorV2 range_del_agg(&bytewise_icmp, snapshots); |
|
||||||
for (const auto& fragment_list : fragment_lists) { |
|
||||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
|
||||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, |
|
||||||
kMaxSequenceNumber)); |
|
||||||
range_del_agg.AddTombstones(std::move(input_iter)); |
|
||||||
} |
|
||||||
|
|
||||||
Slice start("bb"); |
|
||||||
Slice end("e"); |
|
||||||
auto range_del_compaction_iter1 = |
|
||||||
range_del_agg.NewIterator(&start, &end, false /* end_key_inclusive */); |
|
||||||
VerifyFragmentedRangeDels(range_del_compaction_iter1.get(), {{"a", "b", 10}, |
|
||||||
{"b", "c", 20}, |
|
||||||
{"b", "c", 10}, |
|
||||||
{"c", "d", 10}, |
|
||||||
{"c", "d", 8}, |
|
||||||
{"d", "f", 30}, |
|
||||||
{"d", "f", 8}, |
|
||||||
{"f", "g", 8}}); |
|
||||||
|
|
||||||
auto range_del_compaction_iter2 = |
|
||||||
range_del_agg.NewIterator(&start, &end, true /* end_key_inclusive */); |
|
||||||
VerifyFragmentedRangeDels(range_del_compaction_iter2.get(), {{"a", "b", 10}, |
|
||||||
{"b", "c", 20}, |
|
||||||
{"b", "c", 10}, |
|
||||||
{"c", "d", 10}, |
|
||||||
{"c", "d", 8}, |
|
||||||
{"d", "f", 30}, |
|
||||||
{"d", "f", 8}, |
|
||||||
{"f", "g", 8}}); |
|
||||||
} |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
|
|
||||||
int main(int argc, char** argv) { |
|
||||||
::testing::InitGoogleTest(&argc, argv); |
|
||||||
return RUN_ALL_TESTS(); |
|
||||||
} |
|
||||||
Loading…
Reference in new issue