From 343c32be7b0ed3759bc1bcff1eca1e077ee4ec28 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Wed, 26 Feb 2014 16:05:24 -0800 Subject: [PATCH] [CF] DifferentMergeOperators and DifferentCompactionStyles tests Summary: Two new column family tests: * DifferentMergeOperators -- three column families, one without merge operator, one with add operator and one with append operator. verify that operations work as expected. * DifferentCompactionStyles -- three column families, two with level compactions and one with universal compaction. trigger the compactions and verify they work as expected. Test Plan: nope Reviewers: dhruba, haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D16377 --- db/column_family_test.cc | 147 ++++++++++++++++++++++++++++++++++++--- db/db_impl.cc | 2 +- db/db_test.cc | 2 +- 3 files changed, 141 insertions(+), 10 deletions(-) diff --git a/db/column_family_test.cc b/db/column_family_test.cc index 208cb36b3..2843d308c 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -16,6 +16,7 @@ #include "rocksdb/db.h" #include "util/testharness.h" #include "util/testutil.h" +#include "util/coding.h" #include "utilities/merge_operators.h" namespace rocksdb { @@ -140,6 +141,8 @@ class ColumnFamilyTest { ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); } + void WaitForCompaction() { ASSERT_OK(dbfull()->TEST_WaitForCompact()); } + Status Put(int cf, const std::string& key, const std::string& value) { return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(value)); } @@ -163,24 +166,25 @@ class ColumnFamilyTest { return result; } + void CompactAll(int cf) { + ASSERT_OK(db_->CompactRange(handles_[cf], nullptr, nullptr)); + } + void Compact(int cf, const Slice& start, const Slice& limit) { ASSERT_OK(db_->CompactRange(handles_[cf], &start, &limit)); } - int NumTableFilesAtLevel(int cf, int level) { - std::string property; - ASSERT_TRUE(db_->GetProperty( - handles_[cf], "rocksdb.num-files-at-level" + NumberToString(level), - &property)); - return atoi(property.c_str()); + int NumTableFilesAtLevel(int level, int cf) { + return GetProperty(cf, + "rocksdb.num-files-at-level" + std::to_string(level)); } // Return spread of files per level std::string FilesPerLevel(int cf) { std::string result; int last_non_zero_offset = 0; - for (int level = 0; level < column_family_options_.num_levels; level++) { - int f = NumTableFilesAtLevel(cf, level); + for (int level = 0; level < dbfull()->NumberLevels(handles_[cf]); level++) { + int f = NumTableFilesAtLevel(level, cf); char buf[100]; snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); result += buf; @@ -630,6 +634,133 @@ TEST(ColumnFamilyTest, DifferentWriteBufferSizes) { Close(); } +TEST(ColumnFamilyTest, DifferentMergeOperators) { + Open(); + CreateColumnFamilies({"first", "second"}); + ColumnFamilyOptions default_cf, first, second; + first.merge_operator = MergeOperators::CreateUInt64AddOperator(); + second.merge_operator = MergeOperators::CreateStringAppendOperator(); + Reopen({default_cf, first, second}); + + std::string one, two, three; + PutFixed64(&one, 1); + PutFixed64(&two, 2); + PutFixed64(&three, 3); + + ASSERT_OK(Put(0, "foo", two)); + ASSERT_OK(Put(0, "foo", one)); + ASSERT_TRUE(Merge(0, "foo", two).IsNotSupported()); + ASSERT_EQ(Get(0, "foo"), one); + + ASSERT_OK(Put(1, "foo", two)); + ASSERT_OK(Put(1, "foo", one)); + ASSERT_OK(Merge(1, "foo", two)); + ASSERT_EQ(Get(1, "foo"), three); + + ASSERT_OK(Put(2, "foo", two)); + ASSERT_OK(Put(2, "foo", one)); + ASSERT_OK(Merge(2, "foo", two)); + ASSERT_EQ(Get(2, "foo"), one + "," + two); + Close(); +} + +TEST(ColumnFamilyTest, DifferentCompactionStyles) { + Open(); + CreateColumnFamilies({"one", "two"}); + ColumnFamilyOptions default_cf, one, two; + db_options_.max_open_files = 20; // only 10 files in file cache + + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 512; + default_cf.filter_policy = nullptr; + default_cf.no_block_cache = true; + + one.compaction_style = kCompactionStyleUniversal; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 100000; + + two.compaction_style = kCompactionStyleLevel; + two.num_levels = 4; + two.max_mem_compaction_level = 0; + two.level0_file_num_compaction_trigger = 3; + two.write_buffer_size = 100000; + + Reopen({default_cf, one, two}); + + // SETUP column family "default" - test read compaction + ASSERT_EQ("", FilesPerLevel(0)); + PutRandomData(0, 1, 4096); + ASSERT_OK(Flush(0)); + ASSERT_EQ("0,0,1", FilesPerLevel(0)); + // write 8MB + PutRandomData(0, 2000, 4096); + ASSERT_OK(Flush(0)); + // clear levels 0 and 1 + CompactAll(0); + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 0); + // write some new keys into level 0 + PutRandomData(0, 100, 4096); + ASSERT_OK(Flush(0)); + WaitForCompaction(); + // remember number of files in each level + int l1 = NumTableFilesAtLevel(0, 0); + int l2 = NumTableFilesAtLevel(1, 0); + int l3 = NumTableFilesAtLevel(2, 0); + ASSERT_NE(l1, 0); + ASSERT_NE(l2, 0); + ASSERT_NE(l3, 0); + + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger - 1; ++i) { + PutRandomData(1, 12, 10000); + WaitForFlush(1); + ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(1)); + } + + // SETUP column family "two" -- level style with 4 levels + for (int i = 0; i < two.level0_file_num_compaction_trigger - 1; ++i) { + PutRandomData(2, 12, 10000); + WaitForFlush(2); + ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(2)); + } + + // TRIGGER compaction "default" + // read a bunch of times, trigger read compaction + for (int i = 0; i < 200000; ++i) { + Get(0, std::to_string(i)); + } + + // TRIGGER compaction "one" + PutRandomData(1, 12, 10000); + + // TRIGGER compaction "two" + PutRandomData(2, 12, 10000); + + // WAIT for compactions + WaitForCompaction(); + + // VERIFY compaction "default" + // verify that the number of files have decreased + // in some level, indicating that there was a compaction + ASSERT_TRUE(NumTableFilesAtLevel(0, 0) < l1 || + NumTableFilesAtLevel(1, 0) < l2 || + NumTableFilesAtLevel(2, 0) < l3); + + // VERIFY compaction "one" + ASSERT_EQ("1", FilesPerLevel(1)); + + // VERIFY compaction "two" + ASSERT_EQ("0,1", FilesPerLevel(2)); + CompactAll(2); + ASSERT_EQ("0,1", FilesPerLevel(2)); + + Close(); +} + } // namespace rocksdb int main(int argc, char** argv) { diff --git a/db/db_impl.cc b/db/db_impl.cc index 9672c677a..c69ab9da4 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -2668,7 +2668,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact, "compacted to: %s, %.1f MB/sec, level %d, files in(%d, %d) out(%d) " "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) " "write-amplify(%.1f) %s\n", - compact->compaction->input_version()->LevelSummary(&tmp), + cfd->current()->LevelSummary(&tmp), (stats.bytes_readn + stats.bytes_readnp1 + stats.bytes_written) / (double)stats.micros, compact->compaction->output_level(), stats.files_in_leveln, diff --git a/db/db_test.cc b/db/db_test.cc index 70b8439a1..d3a88d7cf 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -4873,7 +4873,7 @@ TEST(DBTest, ReadCompaction) { // remember number of files in each level int l1 = NumTableFilesAtLevel(0, 1); int l2 = NumTableFilesAtLevel(1, 1); - int l3 = NumTableFilesAtLevel(3, 1); + int l3 = NumTableFilesAtLevel(2, 1); ASSERT_NE(NumTableFilesAtLevel(0, 1), 0); ASSERT_NE(NumTableFilesAtLevel(1, 1), 0); ASSERT_NE(NumTableFilesAtLevel(2, 1), 0);