From 7ccd1c80a7ab2d4bda2c5b8d111b8e58373458ed Mon Sep 17 00:00:00 2001 From: sdong Date: Wed, 5 Aug 2015 11:56:19 -0700 Subject: [PATCH] Add two unit tests for SyncWAL() Summary: Add two unit tests for SyncWAL(). One makes sure SyncWAL() doesn't block writes in the other thread. Another one makes sure SyncWAL() doesn't wait ongoing writes to finish before being executed. Create a new test file db_wal_test and move two WAL related tests from db_test to here. Test Plan: Run the new tests Reviewers: IslamAbdelRahman, rven, kradhakrishnan, kolmike, tnovak, yhchiang Reviewed By: yhchiang Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D43605 --- CMakeLists.txt | 1 + Makefile | 4 ++ db/db_test.cc | 51 ------------- db/db_wal_test.cc | 144 +++++++++++++++++++++++++++++++++++++ src.mk | 1 + util/db_test_util.h | 12 +++- util/file_reader_writer.cc | 5 +- 7 files changed, 164 insertions(+), 54 deletions(-) create mode 100644 db/db_wal_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index e9b0a0dae..48a19664e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -263,6 +263,7 @@ set(TESTS db/db_inplace_update_test.cc db/db_log_iter_test.cc db/db_universal_compaction_test.cc + db/db_wal_test.cc db/db_tailing_iter_test.cc db/dbformat_test.cc db/deletefile_test.cc diff --git a/Makefile b/Makefile index 95947d093..2b4b3fc6a 100644 --- a/Makefile +++ b/Makefile @@ -227,6 +227,7 @@ TESTS = \ db_inplace_update_test \ db_tailing_iter_test \ db_universal_compaction_test \ + db_wal_test \ block_hash_index_test \ autovector_test \ column_family_test \ @@ -720,6 +721,9 @@ db_iter_test: db/db_iter_test.o $(LIBOBJECTS) $(TESTHARNESS) db_universal_compaction_test: db/db_universal_compaction_test.o util/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +db_wal_test: db/db_wal_test.o util/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS) + $(AM_LINK) + log_write_bench: util/log_write_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) $(pg) diff --git a/db/db_test.cc b/db/db_test.cc index db4a75d44..db8f03795 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -1802,57 +1802,6 @@ TEST_F(DBTest, IgnoreRecoveredLog) { } while (ChangeOptions(kSkipHashCuckoo)); } -TEST_F(DBTest, RollLog) { - do { - CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); - ASSERT_OK(Put(1, "foo", "v1")); - ASSERT_OK(Put(1, "baz", "v5")); - - ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); - for (int i = 0; i < 10; i++) { - ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); - } - ASSERT_OK(Put(1, "foo", "v4")); - for (int i = 0; i < 10; i++) { - ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); - } - } while (ChangeOptions()); -} - -TEST_F(DBTest, WAL) { - do { - CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); - WriteOptions writeOpt = WriteOptions(); - writeOpt.disableWAL = true; - ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1")); - ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); - - ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); - ASSERT_EQ("v1", Get(1, "foo")); - ASSERT_EQ("v1", Get(1, "bar")); - - writeOpt.disableWAL = false; - ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2")); - writeOpt.disableWAL = true; - ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2")); - - ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); - // Both value's should be present. - ASSERT_EQ("v2", Get(1, "bar")); - ASSERT_EQ("v2", Get(1, "foo")); - - writeOpt.disableWAL = true; - ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3")); - writeOpt.disableWAL = false; - ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3")); - - ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); - // again both values should be present. - ASSERT_EQ("v3", Get(1, "foo")); - ASSERT_EQ("v3", Get(1, "bar")); - } while (ChangeCompactOptions()); -} - TEST_F(DBTest, CheckLock) { do { DB* localdb; diff --git a/db/db_wal_test.cc b/db/db_wal_test.cc new file mode 100644 index 000000000..531021ec4 --- /dev/null +++ b/db/db_wal_test.cc @@ -0,0 +1,144 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "port/stack_trace.h" +#include "util/db_test_util.h" +#if !(defined NDEBUG) || !defined(OS_WIN) +#include "util/sync_point.h" +#endif + +namespace rocksdb { +class DBWALTest : public DBTestBase { + public: + DBWALTest() : DBTestBase("/db_wal_test") {} +}; + +TEST_F(DBWALTest, WAL) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v1", Get(1, "bar")); + + writeOpt.disableWAL = false; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2")); + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + // Both value's should be present. + ASSERT_EQ("v2", Get(1, "bar")); + ASSERT_EQ("v2", Get(1, "foo")); + + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3")); + writeOpt.disableWAL = false; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + // again both values should be present. + ASSERT_EQ("v3", Get(1, "foo")); + ASSERT_EQ("v3", Get(1, "bar")); + } while (ChangeCompactOptions()); +} + +TEST_F(DBWALTest, RollLog) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Put(1, "baz", "v5")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + for (int i = 0; i < 10; i++) { + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + } + ASSERT_OK(Put(1, "foo", "v4")); + for (int i = 0; i < 10; i++) { + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + } + } while (ChangeOptions()); +} + +#if !(defined NDEBUG) || !defined(OS_WIN) +TEST_F(DBWALTest, SyncWALNotBlockWrite) { + Options options = CurrentOptions(); + options.max_write_buffer_number = 4; + DestroyAndReopen(options); + + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("foo5", "bar5")); + + rocksdb::SyncPoint::GetInstance()->LoadDependency({ + {"WritableFileWriter::SyncWithoutFlush:1", + "DBWALTest::SyncWALNotBlockWrite:1"}, + {"DBWALTest::SyncWALNotBlockWrite:2", + "WritableFileWriter::SyncWithoutFlush:2"}, + }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); + + std::thread thread([&]() { ASSERT_OK(db_->SyncWAL()); }); + + TEST_SYNC_POINT("DBWALTest::SyncWALNotBlockWrite:1"); + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_OK(Put("foo3", "bar3")); + FlushOptions fo; + fo.wait = false; + ASSERT_OK(db_->Flush(fo)); + ASSERT_OK(Put("foo4", "bar4")); + + TEST_SYNC_POINT("DBWALTest::SyncWALNotBlockWrite:2"); + + thread.join(); + + ASSERT_EQ(Get("foo1"), "bar1"); + ASSERT_EQ(Get("foo2"), "bar2"); + ASSERT_EQ(Get("foo3"), "bar3"); + ASSERT_EQ(Get("foo4"), "bar4"); + ASSERT_EQ(Get("foo5"), "bar5"); + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBWALTest, SyncWALNotWaitWrite) { + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("foo3", "bar3")); + + rocksdb::SyncPoint::GetInstance()->LoadDependency({ + {"SpecialEnv::WalFile::Append:1", "DBWALTest::SyncWALNotWaitWrite:1"}, + {"DBWALTest::SyncWALNotWaitWrite:2", "SpecialEnv::WalFile::Append:2"}, + }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); + + std::thread thread([&]() { ASSERT_OK(Put("foo2", "bar2")); }); + TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1"); + ASSERT_OK(db_->SyncWAL()); + TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2"); + + thread.join(); + + ASSERT_EQ(Get("foo1"), "bar1"); + ASSERT_EQ(Get("foo2"), "bar2"); + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); +} +#endif +} // namespace rocksdb + +int main(int argc, char** argv) { +#if !(defined NDEBUG) || !defined(OS_WIN) + rocksdb::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +#else + return 0; +#endif +} diff --git a/src.mk b/src.mk index 5744df2f1..f35d81954 100644 --- a/src.mk +++ b/src.mk @@ -174,6 +174,7 @@ TEST_BENCH_SOURCES = \ db/db_log_iter_test.cc \ db/db_universal_compaction_test.cc \ db/db_tailing_iter_test.cc \ + db/db_wal_test.cc \ db/deletefile_test.cc \ db/fault_injection_test.cc \ db/file_indexer_test.cc \ diff --git a/util/db_test_util.h b/util/db_test_util.h index 7830b2d75..72c0b1af2 100644 --- a/util/db_test_util.h +++ b/util/db_test_util.h @@ -206,16 +206,24 @@ class SpecialEnv : public EnvWrapper { WalFile(SpecialEnv* env, unique_ptr&& b) : env_(env), base_(std::move(b)) {} Status Append(const Slice& data) override { +#if !(defined NDEBUG) || !defined(OS_WIN) + TEST_SYNC_POINT("SpecialEnv::WalFile::Append:1"); +#endif + Status s; if (env_->log_write_error_.load(std::memory_order_acquire)) { - return Status::IOError("simulated writer error"); + s = Status::IOError("simulated writer error"); } else { int slowdown = env_->log_write_slowdown_.load(std::memory_order_acquire); if (slowdown > 0) { env_->SleepForMicroseconds(slowdown); } - return base_->Append(data); + s = base_->Append(data); } +#if !(defined NDEBUG) || !defined(OS_WIN) + TEST_SYNC_POINT("SpecialEnv::WalFile::Append:2"); +#endif + return s; } Status Close() override { return base_->Close(); } Status Flush() override { return base_->Flush(); } diff --git a/util/file_reader_writer.cc b/util/file_reader_writer.cc index 179dc77bd..e56d566ef 100644 --- a/util/file_reader_writer.cc +++ b/util/file_reader_writer.cc @@ -174,7 +174,10 @@ Status WritableFileWriter::SyncWithoutFlush(bool use_fsync) { "Can't WritableFileWriter::SyncWithoutFlush() because " "WritableFile::IsSyncThreadSafe() is false"); } - return SyncInternal(use_fsync); + TEST_SYNC_POINT("WritableFileWriter::SyncWithoutFlush:1"); + Status s = SyncInternal(use_fsync); + TEST_SYNC_POINT("WritableFileWriter::SyncWithoutFlush:2"); + return s; } Status WritableFileWriter::SyncInternal(bool use_fsync) {