Introduce RangeDelAggregatorV2 (#4649)
Summary: The old RangeDelAggregator did expensive pre-processing work to create a collapsed, binary-searchable representation of range tombstones. With FragmentedRangeTombstoneIterator, much of this work is now unnecessary. RangeDelAggregatorV2 takes advantage of this by seeking in each iterator to find a covering tombstone in ShouldDelete, while doing minimal work in AddTombstones. The old RangeDelAggregator is still used during flush/compaction for now, though RangeDelAggregatorV2 will support those uses in a future PR. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4649 Differential Revision: D13146964 Pulled By: abhimadan fbshipit-source-id: be29a4c020fc440500c137216fcc1cf529571eb3main
Abhishek Madan
6 years ago
committed by
Facebook Github Bot
parent
ed5aec5ba3
commit
457f77b9ff
@ -0,0 +1,195 @@ |
|||||||
|
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#include "db/range_del_aggregator_v2.h" |
||||||
|
|
||||||
|
#include "db/compaction_iteration_stats.h" |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "db/pinned_iterators_manager.h" |
||||||
|
#include "db/range_del_aggregator.h" |
||||||
|
#include "db/range_tombstone_fragmenter.h" |
||||||
|
#include "db/version_edit.h" |
||||||
|
#include "include/rocksdb/comparator.h" |
||||||
|
#include "include/rocksdb/types.h" |
||||||
|
#include "table/internal_iterator.h" |
||||||
|
#include "table/scoped_arena_iterator.h" |
||||||
|
#include "table/table_builder.h" |
||||||
|
#include "util/heap.h" |
||||||
|
#include "util/kv_map.h" |
||||||
|
#include "util/vector_iterator.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
TruncatedRangeDelIterator::TruncatedRangeDelIterator( |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> iter, |
||||||
|
const InternalKeyComparator* icmp, const InternalKey* smallest, |
||||||
|
const InternalKey* largest) |
||||||
|
: iter_(std::move(iter)), icmp_(icmp) { |
||||||
|
if (smallest != nullptr) { |
||||||
|
pinned_bounds_.emplace_back(); |
||||||
|
auto& parsed_smallest = pinned_bounds_.back(); |
||||||
|
if (!ParseInternalKey(smallest->Encode(), &parsed_smallest)) { |
||||||
|
assert(false); |
||||||
|
} |
||||||
|
smallest_ = &parsed_smallest; |
||||||
|
} |
||||||
|
if (largest != nullptr) { |
||||||
|
pinned_bounds_.emplace_back(); |
||||||
|
auto& parsed_largest = pinned_bounds_.back(); |
||||||
|
if (!ParseInternalKey(largest->Encode(), &parsed_largest)) { |
||||||
|
assert(false); |
||||||
|
} |
||||||
|
if (parsed_largest.type == kTypeRangeDeletion && |
||||||
|
parsed_largest.sequence == kMaxSequenceNumber) { |
||||||
|
// The file boundary has been artificially extended by a range tombstone.
|
||||||
|
// We do not need to adjust largest to properly truncate range
|
||||||
|
// tombstones that extend past the boundary.
|
||||||
|
} else if (parsed_largest.sequence == 0) { |
||||||
|
// The largest key in the sstable has a sequence number of 0. Since we
|
||||||
|
// guarantee that no internal keys with the same user key and sequence
|
||||||
|
// number can exist in a DB, we know that the largest key in this sstable
|
||||||
|
// cannot exist as the smallest key in the next sstable. This further
|
||||||
|
// implies that no range tombstone in this sstable covers largest;
|
||||||
|
// otherwise, the file boundary would have been artificially extended.
|
||||||
|
//
|
||||||
|
// Therefore, we will never truncate a range tombstone at largest, so we
|
||||||
|
// can leave it unchanged.
|
||||||
|
} else { |
||||||
|
// The same user key may straddle two sstable boundaries. To ensure that
|
||||||
|
// the truncated end key can cover the largest key in this sstable, reduce
|
||||||
|
// its sequence number by 1.
|
||||||
|
parsed_largest.sequence -= 1; |
||||||
|
} |
||||||
|
largest_ = &parsed_largest; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
bool TruncatedRangeDelIterator::Valid() const { |
||||||
|
return iter_->Valid() && |
||||||
|
(smallest_ == nullptr || |
||||||
|
icmp_->Compare(*smallest_, iter_->parsed_end_key()) < 0) && |
||||||
|
(largest_ == nullptr || |
||||||
|
icmp_->Compare(iter_->parsed_start_key(), *largest_) < 0); |
||||||
|
} |
||||||
|
|
||||||
|
void TruncatedRangeDelIterator::Next() { iter_->TopNext(); } |
||||||
|
|
||||||
|
void TruncatedRangeDelIterator::Prev() { iter_->TopPrev(); } |
||||||
|
|
||||||
|
// NOTE: target is a user key
|
||||||
|
void TruncatedRangeDelIterator::Seek(const Slice& target) { |
||||||
|
if (largest_ != nullptr && |
||||||
|
icmp_->Compare(*largest_, ParsedInternalKey(target, kMaxSequenceNumber, |
||||||
|
kTypeRangeDeletion)) <= 0) { |
||||||
|
iter_->Invalidate(); |
||||||
|
return; |
||||||
|
} |
||||||
|
iter_->Seek(target); |
||||||
|
} |
||||||
|
|
||||||
|
// NOTE: target is a user key
|
||||||
|
void TruncatedRangeDelIterator::SeekForPrev(const Slice& target) { |
||||||
|
if (smallest_ != nullptr && |
||||||
|
icmp_->Compare(ParsedInternalKey(target, 0, kTypeRangeDeletion), |
||||||
|
*smallest_) < 0) { |
||||||
|
iter_->Invalidate(); |
||||||
|
return; |
||||||
|
} |
||||||
|
iter_->SeekForPrev(target); |
||||||
|
} |
||||||
|
|
||||||
|
void TruncatedRangeDelIterator::SeekToFirst() { iter_->SeekToTopFirst(); } |
||||||
|
|
||||||
|
void TruncatedRangeDelIterator::SeekToLast() { iter_->SeekToTopLast(); } |
||||||
|
|
||||||
|
RangeDelAggregatorV2::RangeDelAggregatorV2(const InternalKeyComparator* icmp, |
||||||
|
SequenceNumber upper_bound) |
||||||
|
: icmp_(icmp), upper_bound_(upper_bound) {} |
||||||
|
|
||||||
|
void RangeDelAggregatorV2::AddTombstones( |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
||||||
|
const InternalKey* smallest, const InternalKey* largest) { |
||||||
|
if (input_iter == nullptr || input_iter->empty()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
if (wrapped_range_del_agg != nullptr) { |
||||||
|
wrapped_range_del_agg->AddTombstones(std::move(input_iter), smallest, |
||||||
|
largest); |
||||||
|
// TODO: this eats the status of the wrapped call; may want to propagate it
|
||||||
|
return; |
||||||
|
} |
||||||
|
iters_.emplace_back(new TruncatedRangeDelIterator(std::move(input_iter), |
||||||
|
icmp_, smallest, largest)); |
||||||
|
} |
||||||
|
|
||||||
|
void RangeDelAggregatorV2::AddUnfragmentedTombstones( |
||||||
|
std::unique_ptr<InternalIterator> input_iter) { |
||||||
|
assert(wrapped_range_del_agg == nullptr); |
||||||
|
if (input_iter == nullptr) { |
||||||
|
return; |
||||||
|
} |
||||||
|
pinned_fragments_.emplace_back(new FragmentedRangeTombstoneList( |
||||||
|
std::move(input_iter), *icmp_, false /* one_time_use */)); |
||||||
|
auto fragmented_iter = new FragmentedRangeTombstoneIterator( |
||||||
|
pinned_fragments_.back().get(), upper_bound_, *icmp_); |
||||||
|
AddTombstones( |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator>(fragmented_iter)); |
||||||
|
} |
||||||
|
|
||||||
|
bool RangeDelAggregatorV2::ShouldDelete(const ParsedInternalKey& parsed, |
||||||
|
RangeDelPositioningMode mode) { |
||||||
|
if (wrapped_range_del_agg != nullptr) { |
||||||
|
return wrapped_range_del_agg->ShouldDelete(parsed, mode); |
||||||
|
} |
||||||
|
// TODO: avoid re-seeking every call
|
||||||
|
for (auto& iter : iters_) { |
||||||
|
iter->Seek(parsed.user_key); |
||||||
|
if (iter->Valid() && icmp_->Compare(iter->start_key(), parsed) <= 0 && |
||||||
|
iter->seq() > parsed.sequence) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
bool RangeDelAggregatorV2::IsRangeOverlapped(const Slice& start, |
||||||
|
const Slice& end) { |
||||||
|
assert(wrapped_range_del_agg == nullptr); |
||||||
|
|
||||||
|
// Set the internal start/end keys so that:
|
||||||
|
// - if start_ikey has the same user key and sequence number as the current
|
||||||
|
// end key, start_ikey will be considered greater; and
|
||||||
|
// - if end_ikey has the same user key and sequence number as the current
|
||||||
|
// start key, end_ikey will be considered greater.
|
||||||
|
ParsedInternalKey start_ikey(start, kMaxSequenceNumber, |
||||||
|
static_cast<ValueType>(0)); |
||||||
|
ParsedInternalKey end_ikey(end, 0, static_cast<ValueType>(0)); |
||||||
|
for (auto& iter : iters_) { |
||||||
|
bool checked_candidate_tombstones = false; |
||||||
|
for (iter->SeekForPrev(start); |
||||||
|
iter->Valid() && icmp_->Compare(iter->start_key(), end_ikey) <= 0; |
||||||
|
iter->Next()) { |
||||||
|
checked_candidate_tombstones = true; |
||||||
|
if (icmp_->Compare(start_ikey, iter->end_key()) < 0 && |
||||||
|
icmp_->Compare(iter->start_key(), end_ikey) <= 0) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (!checked_candidate_tombstones) { |
||||||
|
// Do an additional check for when the end of the range is the begin key
|
||||||
|
// of a tombstone, which we missed earlier since SeekForPrev'ing to the
|
||||||
|
// start was invalid.
|
||||||
|
iter->SeekForPrev(end); |
||||||
|
if (iter->Valid() && icmp_->Compare(start_ikey, iter->end_key()) < 0 && |
||||||
|
icmp_->Compare(iter->start_key(), end_ikey) <= 0) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,134 @@ |
|||||||
|
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <list> |
||||||
|
#include <map> |
||||||
|
#include <set> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "db/compaction_iteration_stats.h" |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "db/pinned_iterators_manager.h" |
||||||
|
#include "db/range_del_aggregator.h" |
||||||
|
#include "db/range_tombstone_fragmenter.h" |
||||||
|
#include "db/version_edit.h" |
||||||
|
#include "include/rocksdb/comparator.h" |
||||||
|
#include "include/rocksdb/types.h" |
||||||
|
#include "table/internal_iterator.h" |
||||||
|
#include "table/scoped_arena_iterator.h" |
||||||
|
#include "table/table_builder.h" |
||||||
|
#include "util/heap.h" |
||||||
|
#include "util/kv_map.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class RangeDelAggregatorV2; |
||||||
|
|
||||||
|
class TruncatedRangeDelIterator { |
||||||
|
public: |
||||||
|
TruncatedRangeDelIterator( |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> iter, |
||||||
|
const InternalKeyComparator* icmp, const InternalKey* smallest, |
||||||
|
const InternalKey* largest); |
||||||
|
|
||||||
|
bool Valid() const; |
||||||
|
|
||||||
|
void Next(); |
||||||
|
void Prev(); |
||||||
|
|
||||||
|
// Seeks to the tombstone with the highest viisble sequence number that covers
|
||||||
|
// target (a user key). If no such tombstone exists, the position will be at
|
||||||
|
// the earliest tombstone that ends after target.
|
||||||
|
void Seek(const Slice& target); |
||||||
|
|
||||||
|
// Seeks to the tombstone with the highest viisble sequence number that covers
|
||||||
|
// target (a user key). If no such tombstone exists, the position will be at
|
||||||
|
// the latest tombstone that starts before target.
|
||||||
|
void SeekForPrev(const Slice& target); |
||||||
|
|
||||||
|
void SeekToFirst(); |
||||||
|
void SeekToLast(); |
||||||
|
|
||||||
|
ParsedInternalKey start_key() const { |
||||||
|
return (smallest_ == nullptr || |
||||||
|
icmp_->Compare(*smallest_, iter_->parsed_start_key()) <= 0) |
||||||
|
? iter_->parsed_start_key() |
||||||
|
: *smallest_; |
||||||
|
} |
||||||
|
|
||||||
|
ParsedInternalKey end_key() const { |
||||||
|
return (largest_ == nullptr || |
||||||
|
icmp_->Compare(iter_->parsed_end_key(), *largest_) <= 0) |
||||||
|
? iter_->parsed_end_key() |
||||||
|
: *largest_; |
||||||
|
} |
||||||
|
|
||||||
|
SequenceNumber seq() const { return iter_->seq(); } |
||||||
|
|
||||||
|
private: |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> iter_; |
||||||
|
const InternalKeyComparator* icmp_; |
||||||
|
const ParsedInternalKey* smallest_ = nullptr; |
||||||
|
const ParsedInternalKey* largest_ = nullptr; |
||||||
|
std::list<ParsedInternalKey> pinned_bounds_; |
||||||
|
}; |
||||||
|
|
||||||
|
class RangeDelAggregatorV2 { |
||||||
|
public: |
||||||
|
RangeDelAggregatorV2(const InternalKeyComparator* icmp, |
||||||
|
SequenceNumber upper_bound); |
||||||
|
|
||||||
|
void AddTombstones( |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
||||||
|
const InternalKey* smallest = nullptr, |
||||||
|
const InternalKey* largest = nullptr); |
||||||
|
|
||||||
|
void AddUnfragmentedTombstones(std::unique_ptr<InternalIterator> input_iter); |
||||||
|
|
||||||
|
bool ShouldDelete(const ParsedInternalKey& parsed, |
||||||
|
RangeDelPositioningMode mode); |
||||||
|
|
||||||
|
bool IsRangeOverlapped(const Slice& start, const Slice& end); |
||||||
|
|
||||||
|
// TODO: no-op for now, but won't be once ShouldDelete leverages positioning
|
||||||
|
// mode and doesn't re-seek every ShouldDelete
|
||||||
|
void InvalidateRangeDelMapPositions() {} |
||||||
|
|
||||||
|
bool IsEmpty() const { return iters_.empty(); } |
||||||
|
bool AddFile(uint64_t file_number) { |
||||||
|
return files_seen_.insert(file_number).second; |
||||||
|
} |
||||||
|
|
||||||
|
// Adaptor method to pass calls through to an old-style RangeDelAggregator.
|
||||||
|
// Will be removed once this new version supports an iterator that can be used
|
||||||
|
// during flush/compaction.
|
||||||
|
RangeDelAggregator* DelegateToRangeDelAggregator( |
||||||
|
const std::vector<SequenceNumber>& snapshots) { |
||||||
|
wrapped_range_del_agg.reset(new RangeDelAggregator( |
||||||
|
*icmp_, snapshots, true /* collapse_deletions */)); |
||||||
|
return wrapped_range_del_agg.get(); |
||||||
|
} |
||||||
|
|
||||||
|
std::unique_ptr<RangeDelIterator> NewIterator() { |
||||||
|
assert(wrapped_range_del_agg != nullptr); |
||||||
|
return wrapped_range_del_agg->NewIterator(); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
const InternalKeyComparator* icmp_; |
||||||
|
SequenceNumber upper_bound_; |
||||||
|
|
||||||
|
std::vector<std::unique_ptr<TruncatedRangeDelIterator>> iters_; |
||||||
|
std::list<std::unique_ptr<FragmentedRangeTombstoneList>> pinned_fragments_; |
||||||
|
std::set<uint64_t> files_seen_; |
||||||
|
|
||||||
|
// TODO: remove once V2 supports exposing tombstone iterators
|
||||||
|
std::unique_ptr<RangeDelAggregator> wrapped_range_del_agg; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,469 @@ |
|||||||
|
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#include "db/range_del_aggregator_v2.h" |
||||||
|
|
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "db/db_test_util.h" |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "db/range_tombstone_fragmenter.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class RangeDelAggregatorV2Test : public testing::Test {}; |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
static auto bytewise_icmp = InternalKeyComparator(BytewiseComparator()); |
||||||
|
|
||||||
|
std::unique_ptr<InternalIterator> MakeRangeDelIter( |
||||||
|
const std::vector<RangeTombstone>& range_dels) { |
||||||
|
std::vector<std::string> keys, values; |
||||||
|
for (const auto& range_del : range_dels) { |
||||||
|
auto key_and_value = range_del.Serialize(); |
||||||
|
keys.push_back(key_and_value.first.Encode().ToString()); |
||||||
|
values.push_back(key_and_value.second.ToString()); |
||||||
|
} |
||||||
|
return std::unique_ptr<test::VectorIterator>( |
||||||
|
new test::VectorIterator(keys, values)); |
||||||
|
} |
||||||
|
|
||||||
|
std::vector<std::unique_ptr<FragmentedRangeTombstoneList>> |
||||||
|
MakeFragmentedTombstoneLists( |
||||||
|
const std::vector<std::vector<RangeTombstone>>& range_dels_list) { |
||||||
|
std::vector<std::unique_ptr<FragmentedRangeTombstoneList>> fragment_lists; |
||||||
|
for (const auto& range_dels : range_dels_list) { |
||||||
|
auto range_del_iter = MakeRangeDelIter(range_dels); |
||||||
|
fragment_lists.emplace_back(new FragmentedRangeTombstoneList( |
||||||
|
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */)); |
||||||
|
} |
||||||
|
return fragment_lists; |
||||||
|
} |
||||||
|
|
||||||
|
struct TruncatedIterScanTestCase { |
||||||
|
ParsedInternalKey start; |
||||||
|
ParsedInternalKey end; |
||||||
|
SequenceNumber seq; |
||||||
|
}; |
||||||
|
|
||||||
|
struct TruncatedIterSeekTestCase { |
||||||
|
Slice target; |
||||||
|
ParsedInternalKey start; |
||||||
|
ParsedInternalKey end; |
||||||
|
SequenceNumber seq; |
||||||
|
bool invalid; |
||||||
|
}; |
||||||
|
|
||||||
|
struct ShouldDeleteTestCase { |
||||||
|
ParsedInternalKey lookup_key; |
||||||
|
bool result; |
||||||
|
}; |
||||||
|
|
||||||
|
struct IsRangeOverlappedTestCase { |
||||||
|
Slice start; |
||||||
|
Slice end; |
||||||
|
bool result; |
||||||
|
}; |
||||||
|
|
||||||
|
ParsedInternalKey UncutEndpoint(const Slice& s) { |
||||||
|
return ParsedInternalKey(s, kMaxSequenceNumber, kTypeRangeDeletion); |
||||||
|
} |
||||||
|
|
||||||
|
ParsedInternalKey InternalValue(const Slice& key, SequenceNumber seq) { |
||||||
|
return ParsedInternalKey(key, seq, kTypeValue); |
||||||
|
} |
||||||
|
|
||||||
|
void VerifyIterator( |
||||||
|
TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, |
||||||
|
const std::vector<TruncatedIterScanTestCase>& expected_range_dels) { |
||||||
|
// Test forward iteration.
|
||||||
|
iter->SeekToFirst(); |
||||||
|
for (size_t i = 0; i < expected_range_dels.size(); i++, iter->Next()) { |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
EXPECT_EQ(0, icmp.Compare(iter->start_key(), expected_range_dels[i].start)); |
||||||
|
EXPECT_EQ(0, icmp.Compare(iter->end_key(), expected_range_dels[i].end)); |
||||||
|
EXPECT_EQ(expected_range_dels[i].seq, iter->seq()); |
||||||
|
} |
||||||
|
EXPECT_FALSE(iter->Valid()); |
||||||
|
|
||||||
|
// Test reverse iteration.
|
||||||
|
iter->SeekToLast(); |
||||||
|
std::vector<TruncatedIterScanTestCase> reverse_expected_range_dels( |
||||||
|
expected_range_dels.rbegin(), expected_range_dels.rend()); |
||||||
|
for (size_t i = 0; i < reverse_expected_range_dels.size(); |
||||||
|
i++, iter->Prev()) { |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
EXPECT_EQ(0, icmp.Compare(iter->start_key(), |
||||||
|
reverse_expected_range_dels[i].start)); |
||||||
|
EXPECT_EQ( |
||||||
|
0, icmp.Compare(iter->end_key(), reverse_expected_range_dels[i].end)); |
||||||
|
EXPECT_EQ(reverse_expected_range_dels[i].seq, iter->seq()); |
||||||
|
} |
||||||
|
EXPECT_FALSE(iter->Valid()); |
||||||
|
} |
||||||
|
|
||||||
|
void VerifySeek(TruncatedRangeDelIterator* iter, |
||||||
|
const InternalKeyComparator& icmp, |
||||||
|
const std::vector<TruncatedIterSeekTestCase>& test_cases) { |
||||||
|
for (const auto& test_case : test_cases) { |
||||||
|
iter->Seek(test_case.target); |
||||||
|
if (test_case.invalid) { |
||||||
|
ASSERT_FALSE(iter->Valid()); |
||||||
|
} else { |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); |
||||||
|
EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); |
||||||
|
EXPECT_EQ(test_case.seq, iter->seq()); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void VerifySeekForPrev( |
||||||
|
TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, |
||||||
|
const std::vector<TruncatedIterSeekTestCase>& test_cases) { |
||||||
|
for (const auto& test_case : test_cases) { |
||||||
|
iter->SeekForPrev(test_case.target); |
||||||
|
if (test_case.invalid) { |
||||||
|
ASSERT_FALSE(iter->Valid()); |
||||||
|
} else { |
||||||
|
ASSERT_TRUE(iter->Valid()); |
||||||
|
EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); |
||||||
|
EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); |
||||||
|
EXPECT_EQ(test_case.seq, iter->seq()); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void VerifyShouldDelete(RangeDelAggregatorV2* range_del_agg, |
||||||
|
const std::vector<ShouldDeleteTestCase>& test_cases) { |
||||||
|
for (const auto& test_case : test_cases) { |
||||||
|
EXPECT_EQ( |
||||||
|
test_case.result, |
||||||
|
range_del_agg->ShouldDelete( |
||||||
|
test_case.lookup_key, RangeDelPositioningMode::kForwardTraversal)); |
||||||
|
} |
||||||
|
for (auto it = test_cases.rbegin(); it != test_cases.rend(); ++it) { |
||||||
|
const auto& test_case = *it; |
||||||
|
EXPECT_EQ( |
||||||
|
test_case.result, |
||||||
|
range_del_agg->ShouldDelete( |
||||||
|
test_case.lookup_key, RangeDelPositioningMode::kBackwardTraversal)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void VerifyIsRangeOverlapped( |
||||||
|
RangeDelAggregatorV2* range_del_agg, |
||||||
|
const std::vector<IsRangeOverlappedTestCase>& test_cases) { |
||||||
|
for (const auto& test_case : test_cases) { |
||||||
|
EXPECT_EQ(test_case.result, |
||||||
|
range_del_agg->IsRangeOverlapped(test_case.start, test_case.end)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, EmptyTruncatedIter) { |
||||||
|
auto range_del_iter = MakeRangeDelIter({}); |
||||||
|
FragmentedRangeTombstoneList fragment_list( |
||||||
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */); |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||||
|
bytewise_icmp)); |
||||||
|
|
||||||
|
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
||||||
|
nullptr); |
||||||
|
|
||||||
|
iter.SeekToFirst(); |
||||||
|
ASSERT_FALSE(iter.Valid()); |
||||||
|
|
||||||
|
iter.SeekToLast(); |
||||||
|
ASSERT_FALSE(iter.Valid()); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, UntruncatedIter) { |
||||||
|
auto range_del_iter = |
||||||
|
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
||||||
|
FragmentedRangeTombstoneList fragment_list( |
||||||
|
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||||
|
bytewise_icmp)); |
||||||
|
|
||||||
|
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
||||||
|
nullptr); |
||||||
|
|
||||||
|
VerifyIterator(&iter, bytewise_icmp, |
||||||
|
{{UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
||||||
|
{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{UncutEndpoint("j"), UncutEndpoint("n"), 4}}); |
||||||
|
|
||||||
|
VerifySeek( |
||||||
|
&iter, bytewise_icmp, |
||||||
|
{{"d", UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
||||||
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"ia", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||||
|
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||||
|
{"", UncutEndpoint("a"), UncutEndpoint("e"), 10}}); |
||||||
|
|
||||||
|
VerifySeekForPrev( |
||||||
|
&iter, bytewise_icmp, |
||||||
|
{{"d", UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
||||||
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"n", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||||
|
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, UntruncatedIterWithSnapshot) { |
||||||
|
auto range_del_iter = |
||||||
|
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
||||||
|
FragmentedRangeTombstoneList fragment_list( |
||||||
|
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(&fragment_list, 9 /* snapshot */, |
||||||
|
bytewise_icmp)); |
||||||
|
|
||||||
|
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
||||||
|
nullptr); |
||||||
|
|
||||||
|
VerifyIterator(&iter, bytewise_icmp, |
||||||
|
{{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{UncutEndpoint("j"), UncutEndpoint("n"), 4}}); |
||||||
|
|
||||||
|
VerifySeek( |
||||||
|
&iter, bytewise_icmp, |
||||||
|
{{"d", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"ia", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||||
|
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||||
|
{"", UncutEndpoint("e"), UncutEndpoint("g"), 8}}); |
||||||
|
|
||||||
|
VerifySeekForPrev( |
||||||
|
&iter, bytewise_icmp, |
||||||
|
{{"d", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||||
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"n", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||||
|
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, TruncatedIter) { |
||||||
|
auto range_del_iter = |
||||||
|
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
||||||
|
FragmentedRangeTombstoneList fragment_list( |
||||||
|
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||||
|
bytewise_icmp)); |
||||||
|
|
||||||
|
InternalKey smallest("d", 7, kTypeValue); |
||||||
|
InternalKey largest("m", 9, kTypeValue); |
||||||
|
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, |
||||||
|
&smallest, &largest); |
||||||
|
|
||||||
|
VerifyIterator(&iter, bytewise_icmp, |
||||||
|
{{InternalValue("d", 7), UncutEndpoint("e"), 10}, |
||||||
|
{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{UncutEndpoint("j"), InternalValue("m", 8), 4}}); |
||||||
|
|
||||||
|
VerifySeek( |
||||||
|
&iter, bytewise_icmp, |
||||||
|
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10}, |
||||||
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"ia", UncutEndpoint("j"), InternalValue("m", 8), 4}, |
||||||
|
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||||
|
{"", InternalValue("d", 7), UncutEndpoint("e"), 10}}); |
||||||
|
|
||||||
|
VerifySeekForPrev( |
||||||
|
&iter, bytewise_icmp, |
||||||
|
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10}, |
||||||
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||||
|
{"n", UncutEndpoint("j"), InternalValue("m", 8), 4}, |
||||||
|
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, SingleIterInAggregator) { |
||||||
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, {"c", "g", 8}}); |
||||||
|
FragmentedRangeTombstoneList fragment_list( |
||||||
|
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||||
|
bytewise_icmp)); |
||||||
|
|
||||||
|
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, kMaxSequenceNumber); |
||||||
|
range_del_agg.AddTombstones(std::move(input_iter)); |
||||||
|
|
||||||
|
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, |
||||||
|
{InternalValue("b", 9), true}, |
||||||
|
{InternalValue("d", 9), true}, |
||||||
|
{InternalValue("e", 7), true}, |
||||||
|
{InternalValue("g", 7), false}}); |
||||||
|
|
||||||
|
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||||
|
{"_", "a", true}, |
||||||
|
{"a", "c", true}, |
||||||
|
{"d", "f", true}, |
||||||
|
{"g", "l", false}}); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, MultipleItersInAggregator) { |
||||||
|
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||||
|
{{{"a", "e", 10}, {"c", "g", 8}}, |
||||||
|
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
||||||
|
|
||||||
|
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, kMaxSequenceNumber); |
||||||
|
for (const auto& fragment_list : fragment_lists) { |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator( |
||||||
|
fragment_list.get(), kMaxSequenceNumber, bytewise_icmp)); |
||||||
|
range_del_agg.AddTombstones(std::move(input_iter)); |
||||||
|
} |
||||||
|
|
||||||
|
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), true}, |
||||||
|
{InternalValue("b", 19), false}, |
||||||
|
{InternalValue("b", 9), true}, |
||||||
|
{InternalValue("d", 9), true}, |
||||||
|
{InternalValue("e", 7), true}, |
||||||
|
{InternalValue("g", 7), false}, |
||||||
|
{InternalValue("h", 24), true}, |
||||||
|
{InternalValue("i", 24), false}, |
||||||
|
{InternalValue("ii", 14), true}, |
||||||
|
{InternalValue("j", 14), false}}); |
||||||
|
|
||||||
|
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||||
|
{"_", "a", true}, |
||||||
|
{"a", "c", true}, |
||||||
|
{"d", "f", true}, |
||||||
|
{"g", "l", true}, |
||||||
|
{"x", "y", false}}); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, MultipleItersInAggregatorWithUpperBound) { |
||||||
|
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||||
|
{{{"a", "e", 10}, {"c", "g", 8}}, |
||||||
|
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
||||||
|
|
||||||
|
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
||||||
|
for (const auto& fragment_list : fragment_lists) { |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(fragment_list.get(), |
||||||
|
19 /* snapshot */, bytewise_icmp)); |
||||||
|
range_del_agg.AddTombstones(std::move(input_iter)); |
||||||
|
} |
||||||
|
|
||||||
|
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, |
||||||
|
{InternalValue("a", 9), true}, |
||||||
|
{InternalValue("b", 9), true}, |
||||||
|
{InternalValue("d", 9), true}, |
||||||
|
{InternalValue("e", 7), true}, |
||||||
|
{InternalValue("g", 7), false}, |
||||||
|
{InternalValue("h", 24), false}, |
||||||
|
{InternalValue("i", 24), false}, |
||||||
|
{InternalValue("ii", 14), true}, |
||||||
|
{InternalValue("j", 14), false}}); |
||||||
|
|
||||||
|
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||||
|
{"_", "a", true}, |
||||||
|
{"a", "c", true}, |
||||||
|
{"d", "f", true}, |
||||||
|
{"g", "l", true}, |
||||||
|
{"x", "y", false}}); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, MultipleTruncatedItersInAggregator) { |
||||||
|
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||||
|
{{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); |
||||||
|
std::vector<std::pair<InternalKey, InternalKey>> iter_bounds = { |
||||||
|
{InternalKey("a", 4, kTypeValue), |
||||||
|
InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||||
|
{InternalKey("m", 20, kTypeValue), |
||||||
|
InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||||
|
{InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; |
||||||
|
|
||||||
|
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
||||||
|
for (size_t i = 0; i < fragment_lists.size(); i++) { |
||||||
|
const auto& fragment_list = fragment_lists[i]; |
||||||
|
const auto& bounds = iter_bounds[i]; |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(fragment_list.get(), |
||||||
|
19 /* snapshot */, bytewise_icmp)); |
||||||
|
range_del_agg.AddTombstones(std::move(input_iter), &bounds.first, |
||||||
|
&bounds.second); |
||||||
|
} |
||||||
|
|
||||||
|
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, |
||||||
|
{InternalValue("a", 9), false}, |
||||||
|
{InternalValue("a", 4), true}, |
||||||
|
{InternalValue("m", 10), false}, |
||||||
|
{InternalValue("m", 9), true}, |
||||||
|
{InternalValue("x", 10), false}, |
||||||
|
{InternalValue("x", 9), false}, |
||||||
|
{InternalValue("x", 5), true}, |
||||||
|
{InternalValue("z", 9), false}}); |
||||||
|
|
||||||
|
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||||
|
{"_", "a", true}, |
||||||
|
{"a", "n", true}, |
||||||
|
{"l", "x", true}, |
||||||
|
{"w", "z", true}, |
||||||
|
{"zzz", "zz", false}, |
||||||
|
{"zz", "zzz", false}}); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(RangeDelAggregatorV2Test, MultipleTruncatedItersInAggregatorSameLevel) { |
||||||
|
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||||
|
{{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); |
||||||
|
std::vector<std::pair<InternalKey, InternalKey>> iter_bounds = { |
||||||
|
{InternalKey("a", 4, kTypeValue), |
||||||
|
InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||||
|
{InternalKey("m", 20, kTypeValue), |
||||||
|
InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||||
|
{InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; |
||||||
|
|
||||||
|
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
||||||
|
|
||||||
|
auto add_iter_to_agg = [&](size_t i) { |
||||||
|
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||||
|
new FragmentedRangeTombstoneIterator(fragment_lists[i].get(), |
||||||
|
19 /* snapshot */, bytewise_icmp)); |
||||||
|
range_del_agg.AddTombstones(std::move(input_iter), &iter_bounds[i].first, |
||||||
|
&iter_bounds[i].second); |
||||||
|
}; |
||||||
|
|
||||||
|
add_iter_to_agg(0); |
||||||
|
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, |
||||||
|
{InternalValue("a", 9), false}, |
||||||
|
{InternalValue("a", 4), true}}); |
||||||
|
|
||||||
|
add_iter_to_agg(1); |
||||||
|
VerifyShouldDelete(&range_del_agg, {{InternalValue("m", 10), false}, |
||||||
|
{InternalValue("m", 9), true}}); |
||||||
|
|
||||||
|
add_iter_to_agg(2); |
||||||
|
VerifyShouldDelete(&range_del_agg, {{InternalValue("x", 10), false}, |
||||||
|
{InternalValue("x", 9), false}, |
||||||
|
{InternalValue("x", 5), true}, |
||||||
|
{InternalValue("z", 9), false}}); |
||||||
|
|
||||||
|
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||||
|
{"_", "a", true}, |
||||||
|
{"a", "n", true}, |
||||||
|
{"l", "x", true}, |
||||||
|
{"w", "z", true}, |
||||||
|
{"zzz", "zz", false}, |
||||||
|
{"zz", "zzz", false}}); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
Loading…
Reference in new issue