Merger test

Summary: I abandoned https://reviews.facebook.net/D18789, but I wrote a good unit test there, so let's check it in. :)

Test Plan: this is test

Reviewers: sdong, yhchiang, ljin

Reviewed By: ljin

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D22827

Makefile

@@ -97,6 +97,7 @@ TESTS = \
 	manual_compaction_test \
 	memenv_test \
 	merge_test \
+	merger_test \
 	redis_test \
 	reduce_levels_test \
 	plain_table_db_test \
@@ -434,6 +435,9 @@ write_controller_test: db/write_controller_test.o $(LIBOBJECTS) $(TESTHARNESS)
 merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
+merger_test: table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
 

table/merger_test.cc

@@ -0,0 +1,197 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include <vector>
+#include <string>
+#include <algorithm>
+
+#include "rocksdb/iterator.h"
+#include "table/merger.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+namespace rocksdb {
+
+class VectorIterator : public Iterator {
+ public:
+  explicit VectorIterator(const std::vector<std::string>& keys)
+      : keys_(keys), current_(keys.size()) {
+    std::sort(keys_.begin(), keys_.end());
+  }
+
+  virtual bool Valid() const { return current_ < keys_.size(); }
+
+  virtual void SeekToFirst() { current_ = 0; }
+  virtual void SeekToLast() { current_ = keys_.size() - 1; }
+
+  virtual void Seek(const Slice& target) {
+    current_ = std::lower_bound(keys_.begin(), keys_.end(), target.ToString()) -
+               keys_.begin();
+  }
+
+  virtual void Next() { current_++; }
+  virtual void Prev() { current_--; }
+
+  virtual Slice key() const { return Slice(keys_[current_]); }
+  virtual Slice value() const { return Slice(); }
+
+  virtual Status status() const { return Status::OK(); }
+
+ private:
+  std::vector<std::string> keys_;
+  size_t current_;
+};
+
+class MergerTest {
+ public:
+  MergerTest()
+      : rnd_(3), merging_iterator_(nullptr), single_iterator_(nullptr) {}
+  ~MergerTest() = default;
+  std::vector<std::string> GenerateStrings(int len, int string_len) {
+    std::vector<std::string> ret;
+    for (int i = 0; i < len; ++i) {
+      ret.push_back(test::RandomHumanReadableString(&rnd_, string_len));
+    }
+    return ret;
+  }
+
+  void AssertEquivalence() {
+    auto a = merging_iterator_.get();
+    auto b = single_iterator_.get();
+    if (!a->Valid()) {
+      ASSERT_TRUE(!b->Valid());
+    } else {
+      ASSERT_TRUE(b->Valid());
+      ASSERT_EQ(b->key().ToString(), a->key().ToString());
+      ASSERT_EQ(b->value().ToString(), a->value().ToString());
+    }
+  }
+
+  void SeekToRandom() { Seek(test::RandomHumanReadableString(&rnd_, 5)); }
+
+  void Seek(std::string target) {
+    merging_iterator_->Seek(target);
+    single_iterator_->Seek(target);
+  }
+
+  void SeekToFirst() {
+    merging_iterator_->SeekToFirst();
+    single_iterator_->SeekToFirst();
+  }
+
+  void SeekToLast() {
+    merging_iterator_->SeekToLast();
+    single_iterator_->SeekToLast();
+  }
+
+  void Next(int times) {
+    for (int i = 0; i < times && merging_iterator_->Valid(); ++i) {
+      AssertEquivalence();
+      merging_iterator_->Next();
+      single_iterator_->Next();
+    }
+    AssertEquivalence();
+  }
+
+  void Prev(int times) {
+    for (int i = 0; i < times && merging_iterator_->Valid(); ++i) {
+      AssertEquivalence();
+      merging_iterator_->Prev();
+      single_iterator_->Prev();
+    }
+    AssertEquivalence();
+  }
+
+  void NextAndPrev(int times) {
+    for (int i = 0; i < times && merging_iterator_->Valid(); ++i) {
+      AssertEquivalence();
+      if (rnd_.OneIn(2)) {
+        merging_iterator_->Prev();
+        single_iterator_->Prev();
+      } else {
+        merging_iterator_->Next();
+        single_iterator_->Next();
+      }
+    }
+    AssertEquivalence();
+  }
+
+  void Generate(size_t num_iterators, size_t strings_per_iterator,
+                size_t letters_per_string) {
+    std::vector<Iterator*> small_iterators;
+    for (size_t i = 0; i < num_iterators; ++i) {
+      auto strings = GenerateStrings(strings_per_iterator, letters_per_string);
+      small_iterators.push_back(new VectorIterator(strings));
+      all_keys_.insert(all_keys_.end(), strings.begin(), strings.end());
+    }
+
+    merging_iterator_.reset(NewMergingIterator(
+        BytewiseComparator(), &small_iterators[0], small_iterators.size()));
+    single_iterator_.reset(new VectorIterator(all_keys_));
+  }
+
+  Random rnd_;
+  std::unique_ptr<Iterator> merging_iterator_;
+  std::unique_ptr<Iterator> single_iterator_;
+  std::vector<std::string> all_keys_;
+};
+
+TEST(MergerTest, SeekToRandomNextTest) {
+  Generate(1000, 50, 50);
+  for (int i = 0; i < 10; ++i) {
+    SeekToRandom();
+    AssertEquivalence();
+    Next(50000);
+  }
+}
+
+TEST(MergerTest, SeekToRandomNextSmallStringsTest) {
+  Generate(1000, 50, 2);
+  for (int i = 0; i < 10; ++i) {
+    SeekToRandom();
+    AssertEquivalence();
+    Next(50000);
+  }
+}
+
+TEST(MergerTest, SeekToRandomPrevTest) {
+  Generate(1000, 50, 50);
+  for (int i = 0; i < 10; ++i) {
+    SeekToRandom();
+    AssertEquivalence();
+    Prev(50000);
+  }
+}
+
+TEST(MergerTest, SeekToRandomRandomTest) {
+  Generate(200, 50, 50);
+  for (int i = 0; i < 3; ++i) {
+    SeekToRandom();
+    AssertEquivalence();
+    NextAndPrev(5000);
+  }
+}
+
+TEST(MergerTest, SeekToFirstTest) {
+  Generate(1000, 50, 50);
+  for (int i = 0; i < 10; ++i) {
+    SeekToFirst();
+    AssertEquivalence();
+    Next(50000);
+  }
+}
+
+TEST(MergerTest, SeekToLastTest) {
+  Generate(1000, 50, 50);
+  for (int i = 0; i < 10; ++i) {
+    SeekToLast();
+    AssertEquivalence();
+    Prev(50000);
+  }
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

util/testutil.cc

@@ -23,6 +23,15 @@ Slice RandomString(Random* rnd, int len, std::string* dst) {
   return Slice(*dst);
 }
 
+extern std::string RandomHumanReadableString(Random* rnd, int len) {
+  std::string ret;
+  ret.resize(len);
+  for (int i = 0; i < len; ++i) {
+    ret[i] = static_cast<char>('a' + rnd->Uniform(26));
+  }
+  return ret;
+}
+
 std::string RandomKey(Random* rnd, int len) {
   // Make sure to generate a wide variety of characters so we
   // test the boundary conditions for short-key optimizations.

util/testutil.h

@@ -21,6 +21,8 @@ namespace test {
 // references the generated data.
 extern Slice RandomString(Random* rnd, int len, std::string* dst);
 
+extern std::string RandomHumanReadableString(Random* rnd, int len);
+
 // Return a random key with the specified length that may contain interesting
 // characters (e.g. \x00, \xff, etc.).
 extern std::string RandomKey(Random* rnd, int len);