From accd3debbbe2b628cf3d9d160597fb6c0efdfefb Mon Sep 17 00:00:00 2001 From: Deon Nicholas Date: Fri, 10 May 2013 10:40:10 -0700 Subject: [PATCH] Implemented StringAppendOperator and unit tests. Summary: Implemented the StringAppendOperator class (subclass of MergeOperator). Found in utilities/merge_operators/string_append/stringappend.{h,cc} It is a rocksdb Merge Operator that supports string/list concatenation with a configurable delimiter. The tests are found in .../stringappend_test.cc. It implements a map : key -> (list of strings), with core operations Append(list_key,val) and Get(list_key). Test Plan: 1. Navigate to your rocksdb repository 2. Execute: make stringappend_test (to compile) 3. Execute: ./stringappend_test (to run the tests) 4. Execute: make all check (to test the ENTIRE rocksdb codebase / regression) Reviewers: haobo, dhruba, zshao Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10737 --- Makefile | 6 +- utilities/.DS_Store | Bin 0 -> 6148 bytes utilities/merge_operators/.DS_Store | Bin 0 -> 6148 bytes .../string_append/stringappend.cc | 57 +++ .../string_append/stringappend.h | 31 ++ .../string_append/stringappend_test.cc | 475 ++++++++++++++++++ 6 files changed, 568 insertions(+), 1 deletion(-) create mode 100644 utilities/.DS_Store create mode 100644 utilities/merge_operators/.DS_Store create mode 100644 utilities/merge_operators/string_append/stringappend.cc create mode 100644 utilities/merge_operators/string_append/stringappend.h create mode 100644 utilities/merge_operators/string_append/stringappend_test.cc diff --git a/Makefile b/Makefile index bfefb71d4..e38659d11 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,8 @@ TESTS = \ write_batch_test \ auto_roll_logger_test \ filelock_test \ - merge_test + merge_test \ + stringappend_test TOOLS = \ sst_dump \ @@ -174,6 +175,9 @@ cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) +stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(CXX) utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) + histogram_test: util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) diff --git a/utilities/.DS_Store b/utilities/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..daeccc094b20ae294ebecc3d166c64a4b84cbe49 GIT binary patch literal 6148 zcmeHKyG{c^3>-s>NED=`++W}iR#EtZd;lPU(m;X^LVXqA#iucT20^|Y0#ZNoaB2iHaNC7GEt$=?Y8r`uMj*0Q< zV2BZbxL`Vr>zE~o%@f34I3_Yfv!oJ}YBge5(wT2n*9*tQq{C|Xu)5i5Lb14==eH<_ z^+ZJ}AO)rhoac7s{r`sk!~8!bX(t7wz`s(!7Mty6%~z`4I(s?qwT*s9_nJ?-8`nW$ mh;~ejcFc{p?}cMx(3uZ9Q9lE&i%bgqwE|z3TNl3o literal 0 HcmV?d00001 diff --git a/utilities/merge_operators/.DS_Store b/utilities/merge_operators/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 +#include +#include "leveldb/slice.h" +#include "leveldb/merge_operator.h" +#include "utilities/merge_operators.h" +#include + +namespace leveldb { + + +// Constructor: also specify the delimiter character. +StringAppendOperator::StringAppendOperator(char delim_char) + : delim_(delim_char) { +} + +// Implementation for the merge operation (concatenates two strings) +void StringAppendOperator::Merge(const Slice& key, + const Slice* existing_value, + const Slice& value, + std::string* new_value, + Logger* logger) const { + + // Clear the *new_value for writing. + assert(new_value); + new_value->clear(); + + if (!existing_value) { + // No existing_value. Set *new_value = value + new_value->assign(value.data(),value.size()); + } else { + // Generic append (existing_value != null). + // Reserve *new_value to correct size, and apply concatenation. + new_value->reserve(existing_value->size() + 1 + value.size()); + new_value->assign(existing_value->data(),existing_value->size()); + new_value->append(1,delim_); + new_value->append(value.data(), value.size()); + } + + return; +} + +const char* StringAppendOperator::Name() const { + return "StringAppendOperator"; +} + +} // namespace leveldb + + + diff --git a/utilities/merge_operators/string_append/stringappend.h b/utilities/merge_operators/string_append/stringappend.h new file mode 100644 index 000000000..1cf11d008 --- /dev/null +++ b/utilities/merge_operators/string_append/stringappend.h @@ -0,0 +1,31 @@ +/** + * A MergeOperator for rocksdb/leveldb that implements string append. + * @author Deon Nicholas (dnicholas@fb.com) + * Copyright 2013 Facebook + */ + +#include "leveldb/merge_operator.h" +#include "leveldb/slice.h" + +namespace leveldb { + +class StringAppendOperator : public MergeOperator { + public: + + StringAppendOperator(char delim_char); /// Constructor: specify delimiter + + virtual void Merge(const Slice& key, + const Slice* existing_value, + const Slice& value, + std::string* new_value, + Logger* logger) const override; + + virtual const char* Name() const override; + + private: + char delim_; // The delimiter is inserted between elements + +}; + +} // namespace leveldb + diff --git a/utilities/merge_operators/string_append/stringappend_test.cc b/utilities/merge_operators/string_append/stringappend_test.cc new file mode 100644 index 000000000..38119c451 --- /dev/null +++ b/utilities/merge_operators/string_append/stringappend_test.cc @@ -0,0 +1,475 @@ +/** + * An persistent map : key -> (list of strings), using rocksdb merge. + * This file is a test-harness / use-case for the StringAppendOperator. + * + * @author Deon Nicholas (dnicholas@fb.com) + * Copyright 2013 Facebook, Inc. +*/ + +#include +#include + +#include "leveldb/db.h" +#include "leveldb/merge_operator.h" +#include "utilities/merge_operators.h" +#include "utilities/merge_operators/string_append/stringappend.h" +#include "util/testharness.h" +#include "util/random.h" + +using namespace leveldb; + +namespace leveldb { + +const std::string kDbName = "/tmp/mergetestdb"; // Path to the database on file system + +// OpenDb opens a (possibly new) rocksdb database with a StringAppendOperator +std::shared_ptr OpenDb(StringAppendOperator* append_op) { + DB* db; + Options options; + options.create_if_missing = true; + options.merge_operator = append_op; + Status s = DB::Open(options, kDbName, &db); + if (!s.ok()) { + std::cerr << s.ToString() << std::endl; + assert(false); + } + return std::shared_ptr(db); +} + +/// StringLists represents a set of string-lists, each with a key-index. +/// Supports Append(list,string) and Get(list) +class StringLists { + public: + + //Constructor: specifies the rocksdb db + StringLists(std::shared_ptr db) + : db_(db), + merge_option_(), + get_option_() { + assert(db); + } + + // Append string val onto the list defined by key; return true on success + bool Append(const std::string& key, const std::string& val){ + Slice valSlice(val.data(),val.size()); + auto s = db_->Merge(merge_option_,key,valSlice); + + if (s.ok()) { + return true; + } else { + std::cerr << "ERROR " << s.ToString() << std::endl; + return false; + } + } + + // Returns the list of strings associated with key (or "" if does not exist) + bool Get(const std::string& key, std::string* const result){ + assert(result != NULL); // we should have a place to store the result + + auto s = db_->Get(get_option_, key, result); + + if (s.ok()) { + return true; + } + + // Either key does not exist, or there is some error. + *result = ""; // Always return empty string (just for convenvtion) + + //NotFound is okay; just return empty (similar to std::map) + //But network or db errors, etc, should fail the test (or at least yell) + if (s.ToString() != "NotFound: "){ + std::cerr << "ERROR " << s.ToString() << std::endl; + } + + // Always return false if s.ok() was not true + return false; + } + + private: + std::shared_ptr db_; + WriteOptions merge_option_; + ReadOptions get_option_; + +}; + +// THE TEST CASES BEGIN HERE + +class StringAppendOperatorTest { }; + +TEST(StringAppendOperatorTest,SimpleTest) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op(','); + auto db = OpenDb(&append_op); + StringLists slists(db); + + slists.Append("k1","v1"); + slists.Append("k1","v2"); + slists.Append("k1","v3"); + + std::string res; + bool status = slists.Get("k1",&res); + + assert(status); + ASSERT_EQ(res,"v1,v2,v3"); +} + +TEST(StringAppendOperatorTest,SimpleDelimiterTest) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op('|'); + auto db = OpenDb(&append_op); + StringLists slists(db); + + slists.Append("k1","v1"); + slists.Append("k1","v2"); + slists.Append("k1","v3"); + + std::string res; + slists.Get("k1",&res); + ASSERT_EQ(res,"v1|v2|v3"); +} + +TEST(StringAppendOperatorTest,OneValueNoDelimiterTest) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op('!'); + auto db = OpenDb(&append_op); + StringLists slists(db); + + slists.Append("random_key","single_val"); + + std::string res; + slists.Get("random_key",&res); + ASSERT_EQ(res,"single_val"); +} + +TEST(StringAppendOperatorTest,VariousKeys) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op('\n'); + auto db = OpenDb(&append_op); + StringLists slists(db); + + slists.Append("c","asdasd"); + slists.Append("a","x"); + slists.Append("b","y"); + slists.Append("a","t"); + slists.Append("a","r"); + slists.Append("b","2"); + slists.Append("c","asdasd"); + + std::string a,b,c; + bool sa,sb,sc; + sa = slists.Get("a",&a); + sb = slists.Get("b",&b); + sc = slists.Get("c",&c); + + assert(sa && sb && sc); // All three keys should have been found + + ASSERT_EQ(a,"x\nt\nr"); + ASSERT_EQ(b,"y\n2"); + ASSERT_EQ(c,"asdasd\nasdasd"); +} + +// Generate semi random keys/words from a small distribution. +TEST(StringAppendOperatorTest,RandomMixGetAppend) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op(' '); + auto db = OpenDb(&append_op); + StringLists slists(db); + + // Generate a list of random keys and values + const int kWordCount = 15; + std::string words[] = {"sdasd","triejf","fnjsdfn","dfjisdfsf","342839", + "dsuha","mabuais","sadajsid","jf9834hf","2d9j89", + "dj9823jd","a","dk02ed2dh","$(jd4h984$(*", "mabz"}; + const int kKeyCount = 6; + std::string keys[] = {"dhaiusdhu","denidw","daisda","keykey","muki", + "shzassdianmd"}; + + // Will store a local copy of all data in order to verify correctness + std::map parallel_copy; + + // Generate a bunch of random queries (Append and Get)! + enum query_t { APPEND_OP, GET_OP, NUM_OPS }; + Random randomGen(1337); //deterministic seed; always get same results! + + const int kNumQueries = 30; + for (int q=0; q 0) { + parallel_copy[key] += " " + word; + } else { + parallel_copy[key] = word; + } + + } else if (query == GET_OP) { + // Assumes that a non-existent key just returns + std::string res; + slists.Get(key,&res); + ASSERT_EQ(res,parallel_copy[key]); + } + + } + +} + +TEST(StringAppendOperatorTest,BIGRandomMixGetAppend) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op(' '); + auto db = OpenDb(&append_op); + StringLists slists(db); + + // Generate a list of random keys and values + const int kWordCount = 15; + std::string words[] = {"sdasd","triejf","fnjsdfn","dfjisdfsf","342839", + "dsuha","mabuais","sadajsid","jf9834hf","2d9j89", + "dj9823jd","a","dk02ed2dh","$(jd4h984$(*", "mabz"}; + const int kKeyCount = 6; + std::string keys[] = {"dhaiusdhu","denidw","daisda","keykey","muki", + "shzassdianmd"}; + + // Will store a local copy of all data in order to verify correctness + std::map parallel_copy; + + // Generate a bunch of random queries (Append and Get)! + enum query_t { APPEND_OP, GET_OP, NUM_OPS }; + Random randomGen(9138204); //deterministic seed; always get same results! + + const int kNumQueries = 1000; + for (int q=0; q 0) { + parallel_copy[key] += " " + word; + } else { + parallel_copy[key] = word; + } + + } else if (query == GET_OP) { + // Assumes that a non-existent key just returns + std::string res; + slists.Get(key,&res); + ASSERT_EQ(res,parallel_copy[key]); + } + + } + +} + + +TEST(StringAppendOperatorTest,PersistentVariousKeys) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op('\n'); + + // Perform the following operations in limited scope + { + auto db = OpenDb(&append_op); + StringLists slists(db); + + slists.Append("c","asdasd"); + slists.Append("a","x"); + slists.Append("b","y"); + slists.Append("a","t"); + slists.Append("a","r"); + slists.Append("b","2"); + slists.Append("c","asdasd"); + + std::string a,b,c; + slists.Get("a",&a); + slists.Get("b",&b); + slists.Get("c",&c); + + ASSERT_EQ(a,"x\nt\nr"); + ASSERT_EQ(b,"y\n2"); + ASSERT_EQ(c,"asdasd\nasdasd"); + } + + // Reopen the database (the previous changes should persist / be remembered) + { + auto db = OpenDb(&append_op); + StringLists slists(db); + + slists.Append("c","bbnagnagsx"); + slists.Append("a","sa"); + slists.Append("b","df"); + slists.Append("a","gh"); + slists.Append("a","jk"); + slists.Append("b","l;"); + slists.Append("c","rogosh"); + + std::string a,b,c; + slists.Get("a",&a); + slists.Get("b",&b); + slists.Get("c",&c); + + ASSERT_EQ(a,"x\nt\nr\nsa\ngh\njk"); + ASSERT_EQ(b,"y\n2\ndf\nl;"); + ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx\nrogosh"); + } +} + +TEST(StringAppendOperatorTest,PersistentFlushAndCompaction) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op('\n'); + + // Perform the following operations in limited scope + { + auto db = OpenDb(&append_op); + StringLists slists(db); + std::string a,b,c; + bool success; + + // Append, Flush, Get + slists.Append("c","asdasd"); + db->Flush(leveldb::FlushOptions()); + success = slists.Get("c",&c); + assert(success == true); + ASSERT_EQ(c,"asdasd"); + + // Append, Flush, Append, Get + slists.Append("a","x"); + slists.Append("b","y"); + db->Flush(leveldb::FlushOptions()); + slists.Append("a","t"); + slists.Append("a","r"); + slists.Append("b","2"); + + success = slists.Get("a",&a); + assert(success == true); + ASSERT_EQ(a,"x\nt\nr"); + + success = slists.Get("b",&b); + assert(success == true); + ASSERT_EQ(b,"y\n2"); + + // Append, Get + success = slists.Append("c","asdasd"); + assert(success); + success = slists.Append("b","monkey"); + assert(success); + + // I omit the "assert(success)" checks here. + slists.Get("a",&a); + slists.Get("b",&b); + slists.Get("c",&c); + + ASSERT_EQ(a,"x\nt\nr"); + ASSERT_EQ(b,"y\n2\nmonkey"); + ASSERT_EQ(c,"asdasd\nasdasd"); + } + + // Reopen the database (the previous changes should persist / be remembered) + { + auto db = OpenDb(&append_op); + StringLists slists(db); + std::string a,b,c; + + // Get (Quick check for persistence of previous database) + slists.Get("a",&a); + ASSERT_EQ(a,"x\nt\nr"); + + //Append, Compact, Get + slists.Append("c","bbnagnagsx"); + slists.Append("a","sa"); + slists.Append("b","df"); + db->CompactRange(nullptr,nullptr); + slists.Get("a",&a); + slists.Get("b",&b); + slists.Get("c",&c); + ASSERT_EQ(a,"x\nt\nr\nsa"); + ASSERT_EQ(b,"y\n2\nmonkey\ndf"); + ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx"); + + // Append, Get + slists.Append("a","gh"); + slists.Append("a","jk"); + slists.Append("b","l;"); + slists.Append("c","rogosh"); + slists.Get("a",&a); + slists.Get("b",&b); + slists.Get("c",&c); + ASSERT_EQ(a,"x\nt\nr\nsa\ngh\njk"); + ASSERT_EQ(b,"y\n2\nmonkey\ndf\nl;"); + ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx\nrogosh"); + + // Compact, Get + db->CompactRange(nullptr,nullptr); + ASSERT_EQ(a,"x\nt\nr\nsa\ngh\njk"); + ASSERT_EQ(b,"y\n2\nmonkey\ndf\nl;"); + ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx\nrogosh"); + + // Append, Flush, Compact, Get + slists.Append("b","afcg"); + db->Flush(leveldb::FlushOptions()); + db->CompactRange(nullptr,nullptr); + slists.Get("b",&b); + ASSERT_EQ(b,"y\n2\nmonkey\ndf\nl;\nafcg"); + } +} + +TEST(StringAppendOperatorTest,SimpleTestNullDelimiter) { + DestroyDB(kDbName, Options()); // Start this test with a fresh DB + + StringAppendOperator append_op('\0'); + auto db = OpenDb(&append_op); + StringLists slists(db); + + slists.Append("k1","v1"); + slists.Append("k1","v2"); + slists.Append("k1","v3"); + + std::string res; + bool status = slists.Get("k1",&res); + assert(status); + + // Construct the desired string. Default constructor doesn't like '\0' chars. + std::string checker("v1,v2,v3"); // Verify that the string is right size. + checker[2] = '\0'; // Use null delimiter instead of comma. + checker[5] = '\0'; + assert(checker.size() == 8); // Verify it is still the correct size + + // Check that the leveldb result string matches the desired string + assert(res.size() == checker.size()); + ASSERT_EQ(res,checker); +} + + +} // namespace leveldb + +int main(int arc, char** argv) { + leveldb::test::RunAllTests(); + return 0; +}