From d5b607a43f2e4be1b53d17fa977e7967c482c141 Mon Sep 17 00:00:00 2001 From: Siying Dong Date: Tue, 28 Feb 2017 17:32:58 -0800 Subject: [PATCH] Make db_wal_test slightly faster Summary: Avoid to run db_wal_test in all the DB test options, and some small changes. Closes https://github.com/facebook/rocksdb/pull/1921 Differential Revision: D4622054 Pulled By: siying fbshipit-source-id: 890fd64 --- db/db_test_util.cc | 30 ++++++++++++++++++++++++++ db/db_test_util.h | 5 ++++- db/db_wal_test.cc | 53 +++++++++++++++++++++++----------------------- 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/db/db_test_util.cc b/db/db_test_util.cc index 75fe129d1..058347e3a 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -191,6 +191,36 @@ bool DBTestBase::ChangeCompactOptions() { } } +// Switch between different WAL settings +bool DBTestBase::ChangeWalOptions() { + if (option_config_ == kDefault) { + option_config_ = kDBLogDir; + Destroy(last_options_); + auto options = CurrentOptions(); + Destroy(options); + options.create_if_missing = true; + TryReopen(options); + return true; + } else if (option_config_ == kDBLogDir) { + option_config_ = kWalDirAndMmapReads; + Destroy(last_options_); + auto options = CurrentOptions(); + Destroy(options); + options.create_if_missing = true; + TryReopen(options); + return true; + } else if (option_config_ == kWalDirAndMmapReads) { + option_config_ = kRecycleLogFiles; + Destroy(last_options_); + auto options = CurrentOptions(); + Destroy(options); + TryReopen(options); + return true; + } else { + return false; + } +} + // Switch between different filter policy // Jump from kDefault to kFilter to kFullFilter bool DBTestBase::ChangeFilterOptions() { diff --git a/db/db_test_util.h b/db/db_test_util.h index 1ae0c736a..5194abc4c 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -671,9 +671,12 @@ class DBTestBase : public testing::Test { // test. Return false if there are no more configurations to test. bool ChangeOptions(int skip_mask = kNoSkip); - // Switch between different compaction styles (we have only 2 now). + // Switch between different compaction styles. bool ChangeCompactOptions(); + // Switch between different WAL-realted options. + bool ChangeWalOptions(); + // Switch between different filter policy // Jump from kDefault to kFilter to kFullFilter bool ChangeFilterOptions(); diff --git a/db/db_wal_test.cc b/db/db_wal_test.cc index cbc69f394..715c5056e 100644 --- a/db/db_wal_test.cc +++ b/db/db_wal_test.cc @@ -51,7 +51,7 @@ TEST_F(DBWALTest, WAL) { // again both values should be present. ASSERT_EQ("v3", Get(1, "foo")); ASSERT_EQ("v3", Get(1, "bar")); - } while (ChangeCompactOptions()); + } while (ChangeWalOptions()); } TEST_F(DBWALTest, RollLog) { @@ -68,7 +68,7 @@ TEST_F(DBWALTest, RollLog) { for (int i = 0; i < 10; i++) { ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); } - } while (ChangeOptions()); + } while (ChangeWalOptions()); } TEST_F(DBWALTest, SyncWALNotBlockWrite) { @@ -151,7 +151,7 @@ TEST_F(DBWALTest, Recover) { ASSERT_EQ("v4", Get(1, "foo")); ASSERT_EQ("v2", Get(1, "bar")); ASSERT_EQ("v5", Get(1, "baz")); - } while (ChangeOptions()); + } while (ChangeWalOptions()); } TEST_F(DBWALTest, RecoverWithTableHandle) { @@ -188,23 +188,22 @@ TEST_F(DBWALTest, RecoverWithTableHandle) { } } } - } while (ChangeOptions()); + } while (ChangeWalOptions()); } TEST_F(DBWALTest, IgnoreRecoveredLog) { std::string backup_logs = dbname_ + "/backup_logs"; - // delete old files in backup_logs directory - env_->CreateDirIfMissing(backup_logs); - std::vector old_files; - env_->GetChildren(backup_logs, &old_files); - for (auto& file : old_files) { - if (file != "." && file != "..") { - env_->DeleteFile(backup_logs + "/" + file); - } - } - do { + // delete old files in backup_logs directory + env_->CreateDirIfMissing(backup_logs); + std::vector old_files; + env_->GetChildren(backup_logs, &old_files); + for (auto& file : old_files) { + if (file != "." && file != "..") { + env_->DeleteFile(backup_logs + "/" + file); + } + } Options options = CurrentOptions(); options.create_if_missing = true; options.merge_operator = MergeOperators::CreateUInt64AddOperator(); @@ -277,7 +276,8 @@ TEST_F(DBWALTest, IgnoreRecoveredLog) { } Status s = TryReopen(options); ASSERT_TRUE(!s.ok()); - } while (ChangeOptions(kSkipHashCuckoo)); + Destroy(options); + } while (ChangeWalOptions()); } TEST_F(DBWALTest, RecoveryWithEmptyLog) { @@ -290,7 +290,7 @@ TEST_F(DBWALTest, RecoveryWithEmptyLog) { ASSERT_OK(Put(1, "foo", "v3")); ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); ASSERT_EQ("v3", Get(1, "foo")); - } while (ChangeOptions()); + } while (ChangeWalOptions()); } #if !(defined NDEBUG) || !defined(OS_WIN) @@ -429,7 +429,7 @@ TEST_F(DBWALTest, GetSortedWalFiles) { ASSERT_OK(Put(1, "foo", "v1")); ASSERT_OK(dbfull()->GetSortedWalFiles(log_files)); ASSERT_EQ(1, log_files.size()); - } while (ChangeOptions()); + } while (ChangeWalOptions()); } TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) { @@ -454,7 +454,7 @@ TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) { } // Check at least the first WAL was cleaned up during the recovery. ASSERT_LT(earliest_log_nums[0], earliest_log_nums[1]); - } while (ChangeOptions()); + } while (ChangeWalOptions()); } TEST_F(DBWALTest, RecoverWithLargeLog) { @@ -481,7 +481,7 @@ TEST_F(DBWALTest, RecoverWithLargeLog) { ASSERT_EQ(std::string(10, '3'), Get(1, "small3")); ASSERT_EQ(std::string(10, '4'), Get(1, "small4")); ASSERT_GT(NumTableFilesAtLevel(0, 1), 1); - } while (ChangeCompactOptions()); + } while (ChangeWalOptions()); } // In https://reviews.facebook.net/D20661 we change @@ -679,9 +679,9 @@ class RecoveryTestHelper { // Starting number for the WAL file name like 00010.log static const int kWALFileOffset = 10; // Keys to be written per WAL file - static const int kKeysPerWALFile = 1024; + static const int kKeysPerWALFile = 133; // Size of the value - static const int kValueSize = 10; + static const int kValueSize = 96; // Create WAL files with values filled in static void FillData(DBWALTest* test, const Options& options, @@ -690,7 +690,7 @@ class RecoveryTestHelper { *count = 0; - shared_ptr table_cache = NewLRUCache(50000, 16); + shared_ptr table_cache = NewLRUCache(50, 0); EnvOptions env_options; WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size); @@ -717,12 +717,13 @@ class RecoveryTestHelper { new log::Writer(std::move(file_writer), current_log_number, db_options.recycle_log_file_num > 0)); + WriteBatch batch; for (int i = 0; i < kKeysPerWALFile; i++) { std::string key = "key" + ToString((*count)++); std::string value = test->DummyString(kValueSize); assert(current_log_writer.get() != nullptr); uint64_t seq = versions->LastSequence() + 1; - WriteBatch batch; + batch.Clear(); batch.Put(key, value); WriteBatchInternal::SetSequence(&batch, seq); current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch)); @@ -773,7 +774,7 @@ class RecoveryTestHelper { if (trunc) { ASSERT_EQ(0, truncate(fname.c_str(), static_cast(size * off))); } else { - InduceCorruption(fname, static_cast(size * off), + InduceCorruption(fname, static_cast(size * off + 8), static_cast(size * len)); } } @@ -792,7 +793,7 @@ class RecoveryTestHelper { ASSERT_EQ(offset, lseek(fd, static_cast(offset), SEEK_SET)); void* buf = alloca(len); - memset(buf, 'a', len); + memset(buf, 'b', len); ASSERT_EQ(len, write(fd, buf, static_cast(len))); close(fd); @@ -808,7 +809,7 @@ TEST_F(DBWALTest, kTolerateCorruptedTailRecords) { const int jend = jstart + RecoveryTestHelper::kWALFilesCount; for (auto trunc : {true, false}) { /* Corruption style */ - for (int i = 0; i < 4; i++) { /* Corruption offset position */ + for (int i = 0; i < 3; i++) { /* Corruption offset position */ for (int j = jstart; j < jend; j++) { /* WAL file */ // Fill data for testing Options options = CurrentOptions();