Introduce RangeDelAggregatorV2 (#4649)
Summary: The old RangeDelAggregator did expensive pre-processing work to create a collapsed, binary-searchable representation of range tombstones. With FragmentedRangeTombstoneIterator, much of this work is now unnecessary. RangeDelAggregatorV2 takes advantage of this by seeking in each iterator to find a covering tombstone in ShouldDelete, while doing minimal work in AddTombstones. The old RangeDelAggregator is still used during flush/compaction for now, though RangeDelAggregatorV2 will support those uses in a future PR. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4649 Differential Revision: D13146964 Pulled By: abhimadan fbshipit-source-id: be29a4c020fc440500c137216fcc1cf529571eb3main
Abhishek Madan
6 years ago
committed by
Facebook Github Bot
parent
ed5aec5ba3
commit
457f77b9ff
@ -0,0 +1,195 @@ |
||||
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include "db/range_del_aggregator_v2.h" |
||||
|
||||
#include "db/compaction_iteration_stats.h" |
||||
#include "db/dbformat.h" |
||||
#include "db/pinned_iterators_manager.h" |
||||
#include "db/range_del_aggregator.h" |
||||
#include "db/range_tombstone_fragmenter.h" |
||||
#include "db/version_edit.h" |
||||
#include "include/rocksdb/comparator.h" |
||||
#include "include/rocksdb/types.h" |
||||
#include "table/internal_iterator.h" |
||||
#include "table/scoped_arena_iterator.h" |
||||
#include "table/table_builder.h" |
||||
#include "util/heap.h" |
||||
#include "util/kv_map.h" |
||||
#include "util/vector_iterator.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
TruncatedRangeDelIterator::TruncatedRangeDelIterator( |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> iter, |
||||
const InternalKeyComparator* icmp, const InternalKey* smallest, |
||||
const InternalKey* largest) |
||||
: iter_(std::move(iter)), icmp_(icmp) { |
||||
if (smallest != nullptr) { |
||||
pinned_bounds_.emplace_back(); |
||||
auto& parsed_smallest = pinned_bounds_.back(); |
||||
if (!ParseInternalKey(smallest->Encode(), &parsed_smallest)) { |
||||
assert(false); |
||||
} |
||||
smallest_ = &parsed_smallest; |
||||
} |
||||
if (largest != nullptr) { |
||||
pinned_bounds_.emplace_back(); |
||||
auto& parsed_largest = pinned_bounds_.back(); |
||||
if (!ParseInternalKey(largest->Encode(), &parsed_largest)) { |
||||
assert(false); |
||||
} |
||||
if (parsed_largest.type == kTypeRangeDeletion && |
||||
parsed_largest.sequence == kMaxSequenceNumber) { |
||||
// The file boundary has been artificially extended by a range tombstone.
|
||||
// We do not need to adjust largest to properly truncate range
|
||||
// tombstones that extend past the boundary.
|
||||
} else if (parsed_largest.sequence == 0) { |
||||
// The largest key in the sstable has a sequence number of 0. Since we
|
||||
// guarantee that no internal keys with the same user key and sequence
|
||||
// number can exist in a DB, we know that the largest key in this sstable
|
||||
// cannot exist as the smallest key in the next sstable. This further
|
||||
// implies that no range tombstone in this sstable covers largest;
|
||||
// otherwise, the file boundary would have been artificially extended.
|
||||
//
|
||||
// Therefore, we will never truncate a range tombstone at largest, so we
|
||||
// can leave it unchanged.
|
||||
} else { |
||||
// The same user key may straddle two sstable boundaries. To ensure that
|
||||
// the truncated end key can cover the largest key in this sstable, reduce
|
||||
// its sequence number by 1.
|
||||
parsed_largest.sequence -= 1; |
||||
} |
||||
largest_ = &parsed_largest; |
||||
} |
||||
} |
||||
|
||||
bool TruncatedRangeDelIterator::Valid() const { |
||||
return iter_->Valid() && |
||||
(smallest_ == nullptr || |
||||
icmp_->Compare(*smallest_, iter_->parsed_end_key()) < 0) && |
||||
(largest_ == nullptr || |
||||
icmp_->Compare(iter_->parsed_start_key(), *largest_) < 0); |
||||
} |
||||
|
||||
void TruncatedRangeDelIterator::Next() { iter_->TopNext(); } |
||||
|
||||
void TruncatedRangeDelIterator::Prev() { iter_->TopPrev(); } |
||||
|
||||
// NOTE: target is a user key
|
||||
void TruncatedRangeDelIterator::Seek(const Slice& target) { |
||||
if (largest_ != nullptr && |
||||
icmp_->Compare(*largest_, ParsedInternalKey(target, kMaxSequenceNumber, |
||||
kTypeRangeDeletion)) <= 0) { |
||||
iter_->Invalidate(); |
||||
return; |
||||
} |
||||
iter_->Seek(target); |
||||
} |
||||
|
||||
// NOTE: target is a user key
|
||||
void TruncatedRangeDelIterator::SeekForPrev(const Slice& target) { |
||||
if (smallest_ != nullptr && |
||||
icmp_->Compare(ParsedInternalKey(target, 0, kTypeRangeDeletion), |
||||
*smallest_) < 0) { |
||||
iter_->Invalidate(); |
||||
return; |
||||
} |
||||
iter_->SeekForPrev(target); |
||||
} |
||||
|
||||
void TruncatedRangeDelIterator::SeekToFirst() { iter_->SeekToTopFirst(); } |
||||
|
||||
void TruncatedRangeDelIterator::SeekToLast() { iter_->SeekToTopLast(); } |
||||
|
||||
RangeDelAggregatorV2::RangeDelAggregatorV2(const InternalKeyComparator* icmp, |
||||
SequenceNumber upper_bound) |
||||
: icmp_(icmp), upper_bound_(upper_bound) {} |
||||
|
||||
void RangeDelAggregatorV2::AddTombstones( |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
||||
const InternalKey* smallest, const InternalKey* largest) { |
||||
if (input_iter == nullptr || input_iter->empty()) { |
||||
return; |
||||
} |
||||
if (wrapped_range_del_agg != nullptr) { |
||||
wrapped_range_del_agg->AddTombstones(std::move(input_iter), smallest, |
||||
largest); |
||||
// TODO: this eats the status of the wrapped call; may want to propagate it
|
||||
return; |
||||
} |
||||
iters_.emplace_back(new TruncatedRangeDelIterator(std::move(input_iter), |
||||
icmp_, smallest, largest)); |
||||
} |
||||
|
||||
void RangeDelAggregatorV2::AddUnfragmentedTombstones( |
||||
std::unique_ptr<InternalIterator> input_iter) { |
||||
assert(wrapped_range_del_agg == nullptr); |
||||
if (input_iter == nullptr) { |
||||
return; |
||||
} |
||||
pinned_fragments_.emplace_back(new FragmentedRangeTombstoneList( |
||||
std::move(input_iter), *icmp_, false /* one_time_use */)); |
||||
auto fragmented_iter = new FragmentedRangeTombstoneIterator( |
||||
pinned_fragments_.back().get(), upper_bound_, *icmp_); |
||||
AddTombstones( |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator>(fragmented_iter)); |
||||
} |
||||
|
||||
bool RangeDelAggregatorV2::ShouldDelete(const ParsedInternalKey& parsed, |
||||
RangeDelPositioningMode mode) { |
||||
if (wrapped_range_del_agg != nullptr) { |
||||
return wrapped_range_del_agg->ShouldDelete(parsed, mode); |
||||
} |
||||
// TODO: avoid re-seeking every call
|
||||
for (auto& iter : iters_) { |
||||
iter->Seek(parsed.user_key); |
||||
if (iter->Valid() && icmp_->Compare(iter->start_key(), parsed) <= 0 && |
||||
iter->seq() > parsed.sequence) { |
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
bool RangeDelAggregatorV2::IsRangeOverlapped(const Slice& start, |
||||
const Slice& end) { |
||||
assert(wrapped_range_del_agg == nullptr); |
||||
|
||||
// Set the internal start/end keys so that:
|
||||
// - if start_ikey has the same user key and sequence number as the current
|
||||
// end key, start_ikey will be considered greater; and
|
||||
// - if end_ikey has the same user key and sequence number as the current
|
||||
// start key, end_ikey will be considered greater.
|
||||
ParsedInternalKey start_ikey(start, kMaxSequenceNumber, |
||||
static_cast<ValueType>(0)); |
||||
ParsedInternalKey end_ikey(end, 0, static_cast<ValueType>(0)); |
||||
for (auto& iter : iters_) { |
||||
bool checked_candidate_tombstones = false; |
||||
for (iter->SeekForPrev(start); |
||||
iter->Valid() && icmp_->Compare(iter->start_key(), end_ikey) <= 0; |
||||
iter->Next()) { |
||||
checked_candidate_tombstones = true; |
||||
if (icmp_->Compare(start_ikey, iter->end_key()) < 0 && |
||||
icmp_->Compare(iter->start_key(), end_ikey) <= 0) { |
||||
return true; |
||||
} |
||||
} |
||||
|
||||
if (!checked_candidate_tombstones) { |
||||
// Do an additional check for when the end of the range is the begin key
|
||||
// of a tombstone, which we missed earlier since SeekForPrev'ing to the
|
||||
// start was invalid.
|
||||
iter->SeekForPrev(end); |
||||
if (iter->Valid() && icmp_->Compare(start_ikey, iter->end_key()) < 0 && |
||||
icmp_->Compare(iter->start_key(), end_ikey) <= 0) { |
||||
return true; |
||||
} |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,134 @@ |
||||
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#pragma once |
||||
|
||||
#include <list> |
||||
#include <map> |
||||
#include <set> |
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "db/compaction_iteration_stats.h" |
||||
#include "db/dbformat.h" |
||||
#include "db/pinned_iterators_manager.h" |
||||
#include "db/range_del_aggregator.h" |
||||
#include "db/range_tombstone_fragmenter.h" |
||||
#include "db/version_edit.h" |
||||
#include "include/rocksdb/comparator.h" |
||||
#include "include/rocksdb/types.h" |
||||
#include "table/internal_iterator.h" |
||||
#include "table/scoped_arena_iterator.h" |
||||
#include "table/table_builder.h" |
||||
#include "util/heap.h" |
||||
#include "util/kv_map.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class RangeDelAggregatorV2; |
||||
|
||||
class TruncatedRangeDelIterator { |
||||
public: |
||||
TruncatedRangeDelIterator( |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> iter, |
||||
const InternalKeyComparator* icmp, const InternalKey* smallest, |
||||
const InternalKey* largest); |
||||
|
||||
bool Valid() const; |
||||
|
||||
void Next(); |
||||
void Prev(); |
||||
|
||||
// Seeks to the tombstone with the highest viisble sequence number that covers
|
||||
// target (a user key). If no such tombstone exists, the position will be at
|
||||
// the earliest tombstone that ends after target.
|
||||
void Seek(const Slice& target); |
||||
|
||||
// Seeks to the tombstone with the highest viisble sequence number that covers
|
||||
// target (a user key). If no such tombstone exists, the position will be at
|
||||
// the latest tombstone that starts before target.
|
||||
void SeekForPrev(const Slice& target); |
||||
|
||||
void SeekToFirst(); |
||||
void SeekToLast(); |
||||
|
||||
ParsedInternalKey start_key() const { |
||||
return (smallest_ == nullptr || |
||||
icmp_->Compare(*smallest_, iter_->parsed_start_key()) <= 0) |
||||
? iter_->parsed_start_key() |
||||
: *smallest_; |
||||
} |
||||
|
||||
ParsedInternalKey end_key() const { |
||||
return (largest_ == nullptr || |
||||
icmp_->Compare(iter_->parsed_end_key(), *largest_) <= 0) |
||||
? iter_->parsed_end_key() |
||||
: *largest_; |
||||
} |
||||
|
||||
SequenceNumber seq() const { return iter_->seq(); } |
||||
|
||||
private: |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> iter_; |
||||
const InternalKeyComparator* icmp_; |
||||
const ParsedInternalKey* smallest_ = nullptr; |
||||
const ParsedInternalKey* largest_ = nullptr; |
||||
std::list<ParsedInternalKey> pinned_bounds_; |
||||
}; |
||||
|
||||
class RangeDelAggregatorV2 { |
||||
public: |
||||
RangeDelAggregatorV2(const InternalKeyComparator* icmp, |
||||
SequenceNumber upper_bound); |
||||
|
||||
void AddTombstones( |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter, |
||||
const InternalKey* smallest = nullptr, |
||||
const InternalKey* largest = nullptr); |
||||
|
||||
void AddUnfragmentedTombstones(std::unique_ptr<InternalIterator> input_iter); |
||||
|
||||
bool ShouldDelete(const ParsedInternalKey& parsed, |
||||
RangeDelPositioningMode mode); |
||||
|
||||
bool IsRangeOverlapped(const Slice& start, const Slice& end); |
||||
|
||||
// TODO: no-op for now, but won't be once ShouldDelete leverages positioning
|
||||
// mode and doesn't re-seek every ShouldDelete
|
||||
void InvalidateRangeDelMapPositions() {} |
||||
|
||||
bool IsEmpty() const { return iters_.empty(); } |
||||
bool AddFile(uint64_t file_number) { |
||||
return files_seen_.insert(file_number).second; |
||||
} |
||||
|
||||
// Adaptor method to pass calls through to an old-style RangeDelAggregator.
|
||||
// Will be removed once this new version supports an iterator that can be used
|
||||
// during flush/compaction.
|
||||
RangeDelAggregator* DelegateToRangeDelAggregator( |
||||
const std::vector<SequenceNumber>& snapshots) { |
||||
wrapped_range_del_agg.reset(new RangeDelAggregator( |
||||
*icmp_, snapshots, true /* collapse_deletions */)); |
||||
return wrapped_range_del_agg.get(); |
||||
} |
||||
|
||||
std::unique_ptr<RangeDelIterator> NewIterator() { |
||||
assert(wrapped_range_del_agg != nullptr); |
||||
return wrapped_range_del_agg->NewIterator(); |
||||
} |
||||
|
||||
private: |
||||
const InternalKeyComparator* icmp_; |
||||
SequenceNumber upper_bound_; |
||||
|
||||
std::vector<std::unique_ptr<TruncatedRangeDelIterator>> iters_; |
||||
std::list<std::unique_ptr<FragmentedRangeTombstoneList>> pinned_fragments_; |
||||
std::set<uint64_t> files_seen_; |
||||
|
||||
// TODO: remove once V2 supports exposing tombstone iterators
|
||||
std::unique_ptr<RangeDelAggregator> wrapped_range_del_agg; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,469 @@ |
||||
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include "db/range_del_aggregator_v2.h" |
||||
|
||||
#include <memory> |
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "db/db_test_util.h" |
||||
#include "db/dbformat.h" |
||||
#include "db/range_tombstone_fragmenter.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class RangeDelAggregatorV2Test : public testing::Test {}; |
||||
|
||||
namespace { |
||||
|
||||
static auto bytewise_icmp = InternalKeyComparator(BytewiseComparator()); |
||||
|
||||
std::unique_ptr<InternalIterator> MakeRangeDelIter( |
||||
const std::vector<RangeTombstone>& range_dels) { |
||||
std::vector<std::string> keys, values; |
||||
for (const auto& range_del : range_dels) { |
||||
auto key_and_value = range_del.Serialize(); |
||||
keys.push_back(key_and_value.first.Encode().ToString()); |
||||
values.push_back(key_and_value.second.ToString()); |
||||
} |
||||
return std::unique_ptr<test::VectorIterator>( |
||||
new test::VectorIterator(keys, values)); |
||||
} |
||||
|
||||
std::vector<std::unique_ptr<FragmentedRangeTombstoneList>> |
||||
MakeFragmentedTombstoneLists( |
||||
const std::vector<std::vector<RangeTombstone>>& range_dels_list) { |
||||
std::vector<std::unique_ptr<FragmentedRangeTombstoneList>> fragment_lists; |
||||
for (const auto& range_dels : range_dels_list) { |
||||
auto range_del_iter = MakeRangeDelIter(range_dels); |
||||
fragment_lists.emplace_back(new FragmentedRangeTombstoneList( |
||||
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */)); |
||||
} |
||||
return fragment_lists; |
||||
} |
||||
|
||||
struct TruncatedIterScanTestCase { |
||||
ParsedInternalKey start; |
||||
ParsedInternalKey end; |
||||
SequenceNumber seq; |
||||
}; |
||||
|
||||
struct TruncatedIterSeekTestCase { |
||||
Slice target; |
||||
ParsedInternalKey start; |
||||
ParsedInternalKey end; |
||||
SequenceNumber seq; |
||||
bool invalid; |
||||
}; |
||||
|
||||
struct ShouldDeleteTestCase { |
||||
ParsedInternalKey lookup_key; |
||||
bool result; |
||||
}; |
||||
|
||||
struct IsRangeOverlappedTestCase { |
||||
Slice start; |
||||
Slice end; |
||||
bool result; |
||||
}; |
||||
|
||||
ParsedInternalKey UncutEndpoint(const Slice& s) { |
||||
return ParsedInternalKey(s, kMaxSequenceNumber, kTypeRangeDeletion); |
||||
} |
||||
|
||||
ParsedInternalKey InternalValue(const Slice& key, SequenceNumber seq) { |
||||
return ParsedInternalKey(key, seq, kTypeValue); |
||||
} |
||||
|
||||
void VerifyIterator( |
||||
TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, |
||||
const std::vector<TruncatedIterScanTestCase>& expected_range_dels) { |
||||
// Test forward iteration.
|
||||
iter->SeekToFirst(); |
||||
for (size_t i = 0; i < expected_range_dels.size(); i++, iter->Next()) { |
||||
ASSERT_TRUE(iter->Valid()); |
||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), expected_range_dels[i].start)); |
||||
EXPECT_EQ(0, icmp.Compare(iter->end_key(), expected_range_dels[i].end)); |
||||
EXPECT_EQ(expected_range_dels[i].seq, iter->seq()); |
||||
} |
||||
EXPECT_FALSE(iter->Valid()); |
||||
|
||||
// Test reverse iteration.
|
||||
iter->SeekToLast(); |
||||
std::vector<TruncatedIterScanTestCase> reverse_expected_range_dels( |
||||
expected_range_dels.rbegin(), expected_range_dels.rend()); |
||||
for (size_t i = 0; i < reverse_expected_range_dels.size(); |
||||
i++, iter->Prev()) { |
||||
ASSERT_TRUE(iter->Valid()); |
||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), |
||||
reverse_expected_range_dels[i].start)); |
||||
EXPECT_EQ( |
||||
0, icmp.Compare(iter->end_key(), reverse_expected_range_dels[i].end)); |
||||
EXPECT_EQ(reverse_expected_range_dels[i].seq, iter->seq()); |
||||
} |
||||
EXPECT_FALSE(iter->Valid()); |
||||
} |
||||
|
||||
void VerifySeek(TruncatedRangeDelIterator* iter, |
||||
const InternalKeyComparator& icmp, |
||||
const std::vector<TruncatedIterSeekTestCase>& test_cases) { |
||||
for (const auto& test_case : test_cases) { |
||||
iter->Seek(test_case.target); |
||||
if (test_case.invalid) { |
||||
ASSERT_FALSE(iter->Valid()); |
||||
} else { |
||||
ASSERT_TRUE(iter->Valid()); |
||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); |
||||
EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); |
||||
EXPECT_EQ(test_case.seq, iter->seq()); |
||||
} |
||||
} |
||||
} |
||||
|
||||
void VerifySeekForPrev( |
||||
TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, |
||||
const std::vector<TruncatedIterSeekTestCase>& test_cases) { |
||||
for (const auto& test_case : test_cases) { |
||||
iter->SeekForPrev(test_case.target); |
||||
if (test_case.invalid) { |
||||
ASSERT_FALSE(iter->Valid()); |
||||
} else { |
||||
ASSERT_TRUE(iter->Valid()); |
||||
EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); |
||||
EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); |
||||
EXPECT_EQ(test_case.seq, iter->seq()); |
||||
} |
||||
} |
||||
} |
||||
|
||||
void VerifyShouldDelete(RangeDelAggregatorV2* range_del_agg, |
||||
const std::vector<ShouldDeleteTestCase>& test_cases) { |
||||
for (const auto& test_case : test_cases) { |
||||
EXPECT_EQ( |
||||
test_case.result, |
||||
range_del_agg->ShouldDelete( |
||||
test_case.lookup_key, RangeDelPositioningMode::kForwardTraversal)); |
||||
} |
||||
for (auto it = test_cases.rbegin(); it != test_cases.rend(); ++it) { |
||||
const auto& test_case = *it; |
||||
EXPECT_EQ( |
||||
test_case.result, |
||||
range_del_agg->ShouldDelete( |
||||
test_case.lookup_key, RangeDelPositioningMode::kBackwardTraversal)); |
||||
} |
||||
} |
||||
|
||||
void VerifyIsRangeOverlapped( |
||||
RangeDelAggregatorV2* range_del_agg, |
||||
const std::vector<IsRangeOverlappedTestCase>& test_cases) { |
||||
for (const auto& test_case : test_cases) { |
||||
EXPECT_EQ(test_case.result, |
||||
range_del_agg->IsRangeOverlapped(test_case.start, test_case.end)); |
||||
} |
||||
} |
||||
|
||||
} // namespace
|
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, EmptyTruncatedIter) { |
||||
auto range_del_iter = MakeRangeDelIter({}); |
||||
FragmentedRangeTombstoneList fragment_list( |
||||
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */); |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||
bytewise_icmp)); |
||||
|
||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
||||
nullptr); |
||||
|
||||
iter.SeekToFirst(); |
||||
ASSERT_FALSE(iter.Valid()); |
||||
|
||||
iter.SeekToLast(); |
||||
ASSERT_FALSE(iter.Valid()); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, UntruncatedIter) { |
||||
auto range_del_iter = |
||||
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
||||
FragmentedRangeTombstoneList fragment_list( |
||||
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||
bytewise_icmp)); |
||||
|
||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
||||
nullptr); |
||||
|
||||
VerifyIterator(&iter, bytewise_icmp, |
||||
{{UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
||||
{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{UncutEndpoint("j"), UncutEndpoint("n"), 4}}); |
||||
|
||||
VerifySeek( |
||||
&iter, bytewise_icmp, |
||||
{{"d", UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"ia", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||
{"", UncutEndpoint("a"), UncutEndpoint("e"), 10}}); |
||||
|
||||
VerifySeekForPrev( |
||||
&iter, bytewise_icmp, |
||||
{{"d", UncutEndpoint("a"), UncutEndpoint("e"), 10}, |
||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"n", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, UntruncatedIterWithSnapshot) { |
||||
auto range_del_iter = |
||||
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
||||
FragmentedRangeTombstoneList fragment_list( |
||||
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(&fragment_list, 9 /* snapshot */, |
||||
bytewise_icmp)); |
||||
|
||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, |
||||
nullptr); |
||||
|
||||
VerifyIterator(&iter, bytewise_icmp, |
||||
{{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{UncutEndpoint("j"), UncutEndpoint("n"), 4}}); |
||||
|
||||
VerifySeek( |
||||
&iter, bytewise_icmp, |
||||
{{"d", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"ia", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||
{"", UncutEndpoint("e"), UncutEndpoint("g"), 8}}); |
||||
|
||||
VerifySeekForPrev( |
||||
&iter, bytewise_icmp, |
||||
{{"d", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"n", UncutEndpoint("j"), UncutEndpoint("n"), 4}, |
||||
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, TruncatedIter) { |
||||
auto range_del_iter = |
||||
MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); |
||||
FragmentedRangeTombstoneList fragment_list( |
||||
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||
bytewise_icmp)); |
||||
|
||||
InternalKey smallest("d", 7, kTypeValue); |
||||
InternalKey largest("m", 9, kTypeValue); |
||||
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, |
||||
&smallest, &largest); |
||||
|
||||
VerifyIterator(&iter, bytewise_icmp, |
||||
{{InternalValue("d", 7), UncutEndpoint("e"), 10}, |
||||
{UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{UncutEndpoint("j"), InternalValue("m", 8), 4}}); |
||||
|
||||
VerifySeek( |
||||
&iter, bytewise_icmp, |
||||
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10}, |
||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"ia", UncutEndpoint("j"), InternalValue("m", 8), 4}, |
||||
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}, |
||||
{"", InternalValue("d", 7), UncutEndpoint("e"), 10}}); |
||||
|
||||
VerifySeekForPrev( |
||||
&iter, bytewise_icmp, |
||||
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10}, |
||||
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8}, |
||||
{"n", UncutEndpoint("j"), InternalValue("m", 8), 4}, |
||||
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}}); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, SingleIterInAggregator) { |
||||
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, {"c", "g", 8}}); |
||||
FragmentedRangeTombstoneList fragment_list( |
||||
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */); |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(&fragment_list, kMaxSequenceNumber, |
||||
bytewise_icmp)); |
||||
|
||||
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, kMaxSequenceNumber); |
||||
range_del_agg.AddTombstones(std::move(input_iter)); |
||||
|
||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, |
||||
{InternalValue("b", 9), true}, |
||||
{InternalValue("d", 9), true}, |
||||
{InternalValue("e", 7), true}, |
||||
{InternalValue("g", 7), false}}); |
||||
|
||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||
{"_", "a", true}, |
||||
{"a", "c", true}, |
||||
{"d", "f", true}, |
||||
{"g", "l", false}}); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, MultipleItersInAggregator) { |
||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
||||
|
||||
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, kMaxSequenceNumber); |
||||
for (const auto& fragment_list : fragment_lists) { |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator( |
||||
fragment_list.get(), kMaxSequenceNumber, bytewise_icmp)); |
||||
range_del_agg.AddTombstones(std::move(input_iter)); |
||||
} |
||||
|
||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), true}, |
||||
{InternalValue("b", 19), false}, |
||||
{InternalValue("b", 9), true}, |
||||
{InternalValue("d", 9), true}, |
||||
{InternalValue("e", 7), true}, |
||||
{InternalValue("g", 7), false}, |
||||
{InternalValue("h", 24), true}, |
||||
{InternalValue("i", 24), false}, |
||||
{InternalValue("ii", 14), true}, |
||||
{InternalValue("j", 14), false}}); |
||||
|
||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||
{"_", "a", true}, |
||||
{"a", "c", true}, |
||||
{"d", "f", true}, |
||||
{"g", "l", true}, |
||||
{"x", "y", false}}); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, MultipleItersInAggregatorWithUpperBound) { |
||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||
{{{"a", "e", 10}, {"c", "g", 8}}, |
||||
{{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); |
||||
|
||||
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
||||
for (const auto& fragment_list : fragment_lists) { |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), |
||||
19 /* snapshot */, bytewise_icmp)); |
||||
range_del_agg.AddTombstones(std::move(input_iter)); |
||||
} |
||||
|
||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, |
||||
{InternalValue("a", 9), true}, |
||||
{InternalValue("b", 9), true}, |
||||
{InternalValue("d", 9), true}, |
||||
{InternalValue("e", 7), true}, |
||||
{InternalValue("g", 7), false}, |
||||
{InternalValue("h", 24), false}, |
||||
{InternalValue("i", 24), false}, |
||||
{InternalValue("ii", 14), true}, |
||||
{InternalValue("j", 14), false}}); |
||||
|
||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||
{"_", "a", true}, |
||||
{"a", "c", true}, |
||||
{"d", "f", true}, |
||||
{"g", "l", true}, |
||||
{"x", "y", false}}); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, MultipleTruncatedItersInAggregator) { |
||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||
{{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); |
||||
std::vector<std::pair<InternalKey, InternalKey>> iter_bounds = { |
||||
{InternalKey("a", 4, kTypeValue), |
||||
InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||
{InternalKey("m", 20, kTypeValue), |
||||
InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||
{InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; |
||||
|
||||
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
||||
for (size_t i = 0; i < fragment_lists.size(); i++) { |
||||
const auto& fragment_list = fragment_lists[i]; |
||||
const auto& bounds = iter_bounds[i]; |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(fragment_list.get(), |
||||
19 /* snapshot */, bytewise_icmp)); |
||||
range_del_agg.AddTombstones(std::move(input_iter), &bounds.first, |
||||
&bounds.second); |
||||
} |
||||
|
||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, |
||||
{InternalValue("a", 9), false}, |
||||
{InternalValue("a", 4), true}, |
||||
{InternalValue("m", 10), false}, |
||||
{InternalValue("m", 9), true}, |
||||
{InternalValue("x", 10), false}, |
||||
{InternalValue("x", 9), false}, |
||||
{InternalValue("x", 5), true}, |
||||
{InternalValue("z", 9), false}}); |
||||
|
||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||
{"_", "a", true}, |
||||
{"a", "n", true}, |
||||
{"l", "x", true}, |
||||
{"w", "z", true}, |
||||
{"zzz", "zz", false}, |
||||
{"zz", "zzz", false}}); |
||||
} |
||||
|
||||
TEST_F(RangeDelAggregatorV2Test, MultipleTruncatedItersInAggregatorSameLevel) { |
||||
auto fragment_lists = MakeFragmentedTombstoneLists( |
||||
{{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); |
||||
std::vector<std::pair<InternalKey, InternalKey>> iter_bounds = { |
||||
{InternalKey("a", 4, kTypeValue), |
||||
InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||
{InternalKey("m", 20, kTypeValue), |
||||
InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, |
||||
{InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; |
||||
|
||||
RangeDelAggregatorV2 range_del_agg(&bytewise_icmp, 19); |
||||
|
||||
auto add_iter_to_agg = [&](size_t i) { |
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> input_iter( |
||||
new FragmentedRangeTombstoneIterator(fragment_lists[i].get(), |
||||
19 /* snapshot */, bytewise_icmp)); |
||||
range_del_agg.AddTombstones(std::move(input_iter), &iter_bounds[i].first, |
||||
&iter_bounds[i].second); |
||||
}; |
||||
|
||||
add_iter_to_agg(0); |
||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, |
||||
{InternalValue("a", 9), false}, |
||||
{InternalValue("a", 4), true}}); |
||||
|
||||
add_iter_to_agg(1); |
||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("m", 10), false}, |
||||
{InternalValue("m", 9), true}}); |
||||
|
||||
add_iter_to_agg(2); |
||||
VerifyShouldDelete(&range_del_agg, {{InternalValue("x", 10), false}, |
||||
{InternalValue("x", 9), false}, |
||||
{InternalValue("x", 5), true}, |
||||
{InternalValue("z", 9), false}}); |
||||
|
||||
VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, |
||||
{"_", "a", true}, |
||||
{"a", "n", true}, |
||||
{"l", "x", true}, |
||||
{"w", "z", true}, |
||||
{"zzz", "zz", false}, |
||||
{"zz", "zzz", false}}); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
Loading…
Reference in new issue