From d916593eadfa838001b2d2991ae182cf5f0ac01e Mon Sep 17 00:00:00 2001 From: Stanislau Hlebik Date: Tue, 15 Jul 2014 16:10:18 -0700 Subject: [PATCH] Add Prev() for merge operator Summary: Implement Prev() with merge operator for DBIterator. Request from mongoDB. Task 4673663. Test Plan: make all check Reviewers: sdong Reviewed By: sdong Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D19743 --- db/db_iter.cc | 315 +++++++++++------ db/db_test.cc | 948 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1164 insertions(+), 99 deletions(-) diff --git a/db/db_iter.cc b/db/db_iter.cc index 1f49b7aa7..8eb9cea4a 100644 --- a/db/db_iter.cc +++ b/db/db_iter.cc @@ -10,6 +10,7 @@ #include "db/db_iter.h" #include #include +#include #include "db/filename.h" #include "db/dbformat.h" @@ -109,9 +110,14 @@ class DBIter: public Iterator { virtual void SeekToLast(); private: + void PrevInternal(); + void FindParseableKey(ParsedInternalKey* ikey, Direction direction); + bool FindValueForCurrentKey(); + bool FindValueForCurrentKeyUsingSeek(); + void FindPrevUserKey(); + void FindNextUserKey(); inline void FindNextUserEntry(bool skipping); void FindNextUserEntryInternal(bool skipping); - void FindPrevUserEntry(); bool ParseKey(ParsedInternalKey* key); void MergeValuesNewToOld(); @@ -133,8 +139,8 @@ class DBIter: public Iterator { SequenceNumber const sequence_; Status status_; - IterKey saved_key_; // == current key when direction_==kReverse - std::string saved_value_; // == current raw value when direction_==kReverse + IterKey saved_key_; + std::string saved_value_; Direction direction_; bool valid_; bool current_entry_is_merged_; @@ -160,20 +166,11 @@ inline bool DBIter::ParseKey(ParsedInternalKey* ikey) { void DBIter::Next() { assert(valid_); - if (direction_ == kReverse) { // Switch directions? + if (direction_ == kReverse) { + FindNextUserKey(); direction_ = kForward; - // iter_ is pointing just before the entries for this->key(), - // so advance into the range of entries for this->key() and then - // use the normal skipping code below. if (!iter_->Valid()) { iter_->SeekToFirst(); - } else { - iter_->Next(); - } - if (!iter_->Valid()) { - valid_ = false; - saved_key_.Clear(); - return; } } @@ -185,7 +182,6 @@ void DBIter::Next() { FindNextUserEntry(true /* skipping the current user key */); } - // PRE: saved_key_ has the current user key if skipping // POST: saved_key_ should have the next user key if valid_, // if the current entry is a result of merge @@ -327,100 +323,228 @@ void DBIter::MergeValuesNewToOld() { void DBIter::Prev() { assert(valid_); + if (direction_ == kForward) { + FindPrevUserKey(); + direction_ = kReverse; + } + PrevInternal(); +} - // Throw an exception now if merge_operator is provided - // TODO: support backward iteration - if (user_merge_operator_) { - Log(logger_, "Prev not supported yet if merge_operator is provided"); - throw std::logic_error("DBIter::Prev backward iteration not supported" - " if merge_operator is provided"); +void DBIter::PrevInternal() { + if (!iter_->Valid()) { + valid_ = false; + return; } - if (direction_ == kForward) { // Switch directions? - // iter_ is pointing at the current entry. Scan backwards until - // the key changes so we can use the normal reverse scanning code. - assert(iter_->Valid()); // Otherwise valid_ would have been false + ParsedInternalKey ikey; + + while (iter_->Valid()) { saved_key_.SetKey(ExtractUserKey(iter_->key())); - while (true) { - iter_->Prev(); + if (FindValueForCurrentKey()) { + valid_ = true; if (!iter_->Valid()) { - valid_ = false; - saved_key_.Clear(); - ClearSavedValue(); return; } - if (user_comparator_->Compare(ExtractUserKey(iter_->key()), - saved_key_.GetKey()) < 0) { - break; + FindParseableKey(&ikey, kReverse); + if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) { + FindPrevUserKey(); } + return; } - direction_ = kReverse; - } + if (!iter_->Valid()) { + break; + } + FindParseableKey(&ikey, kReverse); + if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) { - FindPrevUserEntry(); + FindPrevUserKey(); + } + } + // We haven't found any key - iterator is not valid + assert(!iter_->Valid()); + valid_ = false; } -void DBIter::FindPrevUserEntry() { - assert(direction_ == kReverse); - uint64_t num_skipped = 0; +// This function checks, if the entry with biggest sequence_number <= sequence_ +// is non kTypeDeletion. If it's not, we save value in saved_value_ +bool DBIter::FindValueForCurrentKey() { + assert(iter_->Valid()); + // Contains operands for merge operator. + std::deque operands; + // last entry before merge (could be kTypeDeletion or kTypeValue) + ValueType last_not_merge_type = kTypeDeletion; + ValueType last_key_entry_type = kTypeDeletion; - ValueType value_type = kTypeDeletion; - bool saved_key_valid = true; - if (iter_->Valid()) { - do { - ParsedInternalKey ikey; - if (ParseKey(&ikey) && ikey.sequence <= sequence_) { - if ((value_type != kTypeDeletion) && - user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) < 0) { - // We encountered a non-deleted value in entries for previous keys, - break; - } - value_type = ikey.type; - if (value_type == kTypeDeletion) { - saved_key_.Clear(); - ClearSavedValue(); - saved_key_valid = false; - } else { - Slice raw_value = iter_->value(); - if (saved_value_.capacity() > raw_value.size() + 1048576) { - std::string empty; - swap(empty, saved_value_); - } - saved_key_.SetKey(ExtractUserKey(iter_->key())); - saved_value_.assign(raw_value.data(), raw_value.size()); - } - } else { - // In the case of ikey.sequence > sequence_, we might have already - // iterated to a different user key. - saved_key_valid = false; - } - num_skipped++; - // If we have sequentially iterated via numerous keys and still not - // found the prev user-key, then it is better to seek so that we can - // avoid too many key comparisons. We seek to the first occurence of - // our current key by looking for max sequence number. - if (saved_key_valid && num_skipped > max_skip_) { - num_skipped = 0; - std::string last_key; - AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetKey(), - kMaxSequenceNumber, - kValueTypeForSeek)); - iter_->Seek(last_key); - RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + ParsedInternalKey ikey; + FindParseableKey(&ikey, kReverse); + + size_t num_skipped = 0; + while (iter_->Valid() && ikey.sequence <= sequence_ && + (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0)) { + // We iterate too much: let's use Seek() to avoid too much key comparisons + if (num_skipped >= max_skip_) { + return FindValueForCurrentKeyUsingSeek(); + } + + last_key_entry_type = ikey.type; + switch (last_key_entry_type) { + case kTypeValue: + operands.clear(); + saved_value_ = iter_->value().ToString(); + last_not_merge_type = kTypeValue; + break; + case kTypeDeletion: + operands.clear(); + last_not_merge_type = kTypeDeletion; + break; + case kTypeMerge: + assert(user_merge_operator_ != nullptr); + operands.push_back(iter_->value().ToString()); + break; + default: + assert(false); + } + + assert(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0); + iter_->Prev(); + ++num_skipped; + FindParseableKey(&ikey, kReverse); + } + + switch (last_key_entry_type) { + case kTypeDeletion: + valid_ = false; + return false; + case kTypeMerge: + if (last_not_merge_type == kTypeDeletion) { + user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands, + &saved_value_, logger_); } else { - iter_->Prev(); + assert(last_not_merge_type == kTypeValue); + std::string last_put_value = saved_value_; + Slice temp_slice(last_put_value); + user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice, + operands, &saved_value_, logger_); } - } while (iter_->Valid()); + break; + case kTypeValue: + // do nothing - we've already has value in saved_value_ + break; + default: + assert(false); + break; } + valid_ = true; + return true; +} - if (value_type == kTypeDeletion) { - // End +// This function is used in FindValueForCurrentKey. +// We use Seek() function instead of Prev() to find necessary value +bool DBIter::FindValueForCurrentKeyUsingSeek() { + std::string last_key; + AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetKey(), sequence_, + kValueTypeForSeek)); + iter_->Seek(last_key); + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + + // assume there is at least one parseable key for this user key + ParsedInternalKey ikey; + FindParseableKey(&ikey, kForward); + + if (ikey.type == kTypeValue || ikey.type == kTypeDeletion) { + if (ikey.type == kTypeValue) { + saved_value_ = iter_->value().ToString(); + valid_ = true; + return true; + } valid_ = false; - saved_key_.Clear(); - ClearSavedValue(); - direction_ = kForward; - } else { + return false; + } + + // kTypeMerge. We need to collect all kTypeMerge values and save them + // in operands + std::deque operands; + while (iter_->Valid() && + (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) && + ikey.type == kTypeMerge) { + operands.push_front(iter_->value().ToString()); + iter_->Next(); + FindParseableKey(&ikey, kForward); + } + + if (!iter_->Valid() || + (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0) || + ikey.type == kTypeDeletion) { + user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands, + &saved_value_, logger_); + + // Make iter_ valid and point to saved_key_ + if (!iter_->Valid() || + (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0)) { + iter_->Seek(last_key); + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + } valid_ = true; + return true; + } + + const Slice& value = iter_->value(); + user_merge_operator_->FullMerge(saved_key_.GetKey(), &value, operands, + &saved_value_, logger_); + valid_ = true; + return true; +} + +// Used in Next to change directions +// Go to next user key +// Don't use Seek(), +// because next user key will be very close +void DBIter::FindNextUserKey() { + if (!iter_->Valid()) { + return; + } + ParsedInternalKey ikey; + FindParseableKey(&ikey, kForward); + while (iter_->Valid() && + user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0) { + iter_->Next(); + FindParseableKey(&ikey, kForward); + } +} + +// Go to previous user_key +void DBIter::FindPrevUserKey() { + if (!iter_->Valid()) { + return; + } + size_t num_skipped = 0; + ParsedInternalKey ikey; + FindParseableKey(&ikey, kReverse); + while (iter_->Valid() && + user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) { + if (num_skipped >= max_skip_) { + num_skipped = 0; + IterKey last_key; + last_key.SetInternalKey(ParsedInternalKey( + saved_key_.GetKey(), kMaxSequenceNumber, kValueTypeForSeek)); + iter_->Seek(last_key.GetKey()); + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + } + + iter_->Prev(); + ++num_skipped; + FindParseableKey(&ikey, kReverse); + } +} + +// Skip all unparseable keys +void DBIter::FindParseableKey(ParsedInternalKey* ikey, Direction direction) { + while (iter_->Valid() && !ParseKey(ikey)) { + if (direction == kReverse) { + iter_->Prev(); + } else { + iter_->Next(); + } } } @@ -454,20 +578,13 @@ void DBIter::SeekToFirst() { } void DBIter::SeekToLast() { - // Throw an exception for now if merge_operator is provided - // TODO: support backward iteration - if (user_merge_operator_) { - Log(logger_, "SeekToLast not supported yet if merge_operator is provided"); - throw std::logic_error("DBIter::SeekToLast: backward iteration not" - " supported if merge_operator is provided"); - } - direction_ = kReverse; ClearSavedValue(); PERF_TIMER_AUTO(seek_internal_seek_time); iter_->SeekToLast(); PERF_TIMER_STOP(seek_internal_seek_time); - FindPrevUserEntry(); + + PrevInternal(); } Iterator* NewDBIterator(Env* env, const Options& options, diff --git a/db/db_test.cc b/db/db_test.cc index 3150f8e08..dd6a4d051 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -12,8 +12,10 @@ #include #include #include +#include #include "db/dbformat.h" +#include "db/db_iter.h" #include "db/db_impl.h" #include "db/filename.h" #include "db/version_set.h" @@ -1162,6 +1164,952 @@ TEST(DBTest, ReadOnlyDB) { delete iter; } +class TestIterator : public Iterator { + public: + explicit TestIterator(const Comparator* comparator) + : initialized_(false), + valid_(false), + sequence_number_(0), + iter_(0), + cmp(comparator) {} + + void AddMerge(std::string key, std::string value) { + Add(key, kTypeMerge, value); + } + + void AddDeletion(std::string key) { Add(key, kTypeDeletion, std::string()); } + + void AddPut(std::string key, std::string value) { + Add(key, kTypeValue, value); + } + + void Add(std::string key, ValueType type, std::string value) { + valid_ = true; + ParsedInternalKey internal_key(key, sequence_number_++, type); + data_.push_back(std::pair(std::string(), value)); + AppendInternalKey(&data_.back().first, internal_key); + } + + // should be called before operations with iterator + void Finish() { + initialized_ = true; + std::sort(data_.begin(), data_.end(), + [this](std::pair a, + std::pair b) { + return (cmp.Compare(a.first, b.first) < 0); + }); + } + + virtual bool Valid() const override { + assert(initialized_); + return valid_; + } + + virtual void SeekToFirst() override { + assert(initialized_); + valid_ = (data_.size() > 0); + iter_ = 0; + } + + virtual void SeekToLast() override { + assert(initialized_); + valid_ = (data_.size() > 0); + iter_ = data_.size() - 1; + } + + virtual void Seek(const Slice& target) override { + assert(initialized_); + SeekToFirst(); + if (!valid_) { + return; + } + while (iter_ < data_.size() && + (cmp.Compare(data_[iter_].first, target) < 0)) { + ++iter_; + } + + if (iter_ == data_.size()) { + valid_ = false; + } + } + + virtual void Next() override { + assert(initialized_); + if (data_.empty() || (iter_ == data_.size() - 1)) { + valid_ = false; + } else { + ++iter_; + } + } + + virtual void Prev() override { + assert(initialized_); + if (iter_ == 0) { + valid_ = false; + } else { + --iter_; + } + } + + virtual Slice key() const override { + assert(initialized_); + return data_[iter_].first; + } + + virtual Slice value() const override { + assert(initialized_); + return data_[iter_].second; + } + + virtual Status status() const override { + assert(initialized_); + return Status::OK(); + } + + private: + bool initialized_; + bool valid_; + size_t sequence_number_; + size_t iter_; + + InternalKeyComparator cmp; + std::vector> data_; +}; + +TEST(DBTest, DBIteratorPrevNext) { + Options options; + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddDeletion("a"); + internal_iter.AddDeletion("a"); + internal_iter.AddDeletion("a"); + internal_iter.AddDeletion("a"); + internal_iter.AddPut("a", "val_a"); + + internal_iter.AddPut("b", "val_b"); + internal_iter.Finish(); + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + Options options; + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("b", "val_b"); + + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("b", "val_b"); + + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("b", "val_b"); + + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("b", "val_b"); + + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("b", "val_b"); + internal_iter.Finish(); + + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + } + + { + Options options; + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("a", "val_a"); + internal_iter.AddPut("a", "val_a"); + + internal_iter.AddPut("b", "val_b"); + + internal_iter.AddPut("c", "val_c"); + internal_iter.Finish(); + + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + } +} + +TEST(DBTest, DBIteratorEmpty) { + Options options = CurrentOptions(); + TestIterator internal_iter(BytewiseComparator()); + internal_iter.Finish(); + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0); + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0); + db_iter->SeekToFirst(); + ASSERT_TRUE(!db_iter->Valid()); + } +} + +TEST(DBTest, DBIteratorUseSkipCountSkips) { + Options options = CurrentOptions(); + options.statistics = rocksdb::CreateDBStatistics(); + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator internal_iter(BytewiseComparator()); + for (size_t i = 0; i < 200; ++i) { + internal_iter.AddPut("a", "a"); + internal_iter.AddPut("b", "b"); + internal_iter.AddPut("c", "c"); + } + internal_iter.Finish(); + + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "c"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "b"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "a"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3); +} + +TEST(DBTest, DBIteratorUseSkip) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + { + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddMerge("b", "merge_1"); + internal_iter.AddMerge("a", "merge_2"); + for (size_t i = 0; i < 200; ++i) { + internal_iter.AddPut("c", std::to_string(i)); + } + internal_iter.Finish(); + + for (size_t i = 0; i < 200; ++i) { + options.statistics = rocksdb::CreateDBStatistics(); + auto db_iter = NewDBIterator(env_, options, BytewiseComparator(), + &internal_iter, i + 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), std::to_string(i)); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + } + } + + { + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddMerge("b", "merge_1"); + internal_iter.AddMerge("a", "merge_2"); + for (size_t i = 0; i < 200; ++i) { + internal_iter.AddDeletion("c"); + } + internal_iter.AddPut("c", "200"); + internal_iter.Finish(); + + for (size_t i = 0; i < 200; ++i) { + auto db_iter = NewDBIterator(env_, options, BytewiseComparator(), + &internal_iter, i + 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = NewDBIterator(env_, options, BytewiseComparator(), + &internal_iter, 202); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "200"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + } + } + + { + TestIterator internal_iter(BytewiseComparator()); + for (size_t i = 0; i < 200; ++i) { + internal_iter.AddDeletion("c"); + } + internal_iter.AddPut("c", "200"); + internal_iter.Finish(); + + for (size_t i = 0; i < 200; ++i) { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, i); + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + + db_iter->SeekToFirst(); + ASSERT_TRUE(!db_iter->Valid()); + } + + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 200); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "200"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "200"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddMerge("b", "merge_1"); + internal_iter.AddMerge("a", "merge_2"); + for (size_t i = 0; i < 200; ++i) { + internal_iter.AddPut("d", std::to_string(i)); + } + + for (size_t i = 0; i < 200; ++i) { + internal_iter.AddPut("c", std::to_string(i)); + } + internal_iter.Finish(); + + for (size_t i = 0; i < 200; ++i) { + auto db_iter = NewDBIterator(env_, options, BytewiseComparator(), + &internal_iter, i + 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), std::to_string(i)); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + } + } + + { + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddMerge("b", "b"); + internal_iter.AddMerge("a", "a"); + for (size_t i = 0; i < 200; ++i) { + internal_iter.AddMerge("c", std::to_string(i)); + } + internal_iter.Finish(); + + for (size_t i = 0; i < 200; ++i) { + auto db_iter = NewDBIterator(env_, options, BytewiseComparator(), + &internal_iter, i + 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + std::string merge_result = "0"; + for (size_t j = 1; j <= i; ++j) { + merge_result += "," + std::to_string(j); + } + ASSERT_EQ(db_iter->value().ToString(), merge_result); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "b"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "a"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + } +} + +TEST(DBTest, DBIterator) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddPut("a", "0"); + internal_iter.AddPut("b", "0"); + internal_iter.AddDeletion("b"); + internal_iter.AddMerge("a", "1"); + internal_iter.AddMerge("b", "2"); + internal_iter.Finish(); + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 1); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0,1"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "2"); + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddMerge("a", "merge_1"); + internal_iter.AddMerge("a", "merge_2"); + internal_iter.AddMerge("a", "merge_3"); + internal_iter.AddPut("a", "put_1"); + internal_iter.AddMerge("a", "merge_4"); + internal_iter.AddMerge("a", "merge_5"); + internal_iter.AddMerge("a", "merge_6"); + internal_iter.Finish(); + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 1); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 3); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 5); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 6); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5,merge_6"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + } + + { + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddMerge("a", "merge_1"); + internal_iter.AddMerge("a", "merge_2"); + internal_iter.AddMerge("a", "merge_3"); + internal_iter.AddDeletion("a"); + internal_iter.AddMerge("a", "merge_4"); + internal_iter.AddMerge("a", "merge_5"); + internal_iter.AddMerge("a", "merge_6"); + internal_iter.Finish(); + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 1); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 3); + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 5); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 6); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5,merge_6"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + } + + { + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddMerge("a", "merge_1"); + internal_iter.AddPut("b", "val"); + internal_iter.AddMerge("b", "merge_2"); + + internal_iter.AddDeletion("b"); + internal_iter.AddMerge("b", "merge_3"); + + internal_iter.AddMerge("c", "merge_4"); + internal_iter.AddMerge("c", "merge_5"); + + internal_iter.AddDeletion("b"); + internal_iter.AddMerge("b", "merge_6"); + internal_iter.AddMerge("b", "merge_7"); + internal_iter.AddMerge("b", "merge_8"); + internal_iter.AddMerge("b", "merge_9"); + internal_iter.AddMerge("b", "merge_10"); + internal_iter.AddMerge("b", "merge_11"); + + internal_iter.AddDeletion("c"); + internal_iter.Finish(); + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 0); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 2); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val,merge_2"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 4); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 5); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4"); + db_iter->Prev(); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 6); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 7); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 9); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_6,merge_7"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = NewDBIterator(env_, options, BytewiseComparator(), + &internal_iter, 13); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), + "merge_6,merge_7,merge_8,merge_9,merge_10,merge_11"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + auto db_iter = NewDBIterator(env_, options, BytewiseComparator(), + &internal_iter, 14); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), + "merge_6,merge_7,merge_8,merge_9,merge_10,merge_11"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + } + + { + Options options = CurrentOptions(); + TestIterator internal_iter(BytewiseComparator()); + internal_iter.AddDeletion("a"); + internal_iter.AddPut("a", "0"); + internal_iter.AddPut("b", "0"); + internal_iter.Finish(); + + auto db_iter = + NewDBIterator(env_, options, BytewiseComparator(), &internal_iter, 10); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + } +} + // Make sure that when options.block_cache is set, after a new table is // created its index/filter blocks are added to block cache. TEST(DBTest, IndexAndFilterBlocksOfNewTableAddedToCache) {