From 58547e533b39d30cb2a6855084b6313e38752d71 Mon Sep 17 00:00:00 2001 From: sdong Date: Mon, 29 Jun 2020 16:55:35 -0700 Subject: [PATCH] Disable fsync in some tests to speed them up (#7036) Summary: Fsyncing files is not providing more test coverage in many tests. Provide an option in SpecialEnv to turn it off to speed it up and enable this option in some tests with relatively long run time. Most of those tests can be divided as parameterized gtest too. This two speed up approaches are orthogonal and we can do both if needed. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7036 Test Plan: Run all tests and make sure they pass. Reviewed By: ltamasi Differential Revision: D22268084 fbshipit-source-id: 6d4a838a1b7328c13931a2a5d93de57aa02afaab --- db/db_test.cc | 4 +++ db/db_test_util.h | 64 ++++++++++++++++++++++++++++++++++-- db/external_sst_file_test.cc | 2 ++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/db/db_test.cc b/db/db_test.cc index 0c932c3c1..75ba20a16 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -1788,6 +1788,7 @@ TEST_F(DBTest, Snapshot) { TEST_F(DBTest, HiddenValuesAreRemoved) { anon::OptionsOverride options_override; options_override.skip_policy = kSkipNoSnapshot; + env_->skip_fsync_ = true; do { Options options = CurrentOptions(options_override); CreateAndReopenWithCF({"pikachu"}, options); @@ -3985,6 +3986,7 @@ TEST_F(DBTest, DynamicMemtableOptions) { const uint64_t k128KB = 1 << 17; const uint64_t k5KB = 5 * 1024; Options options; + env_->skip_fsync_ = true; options.env = env_; options.create_if_missing = true; options.compression = kNoCompression; @@ -5131,6 +5133,7 @@ TEST_F(DBTest, DynamicUniversalCompactionOptions) { TEST_F(DBTest, FileCreationRandomFailure) { Options options; + env_->skip_fsync_ = true; options.env = env_; options.create_if_missing = true; options.write_buffer_size = 100000; // Small write buffer @@ -5490,6 +5493,7 @@ TEST_F(DBTest, MergeTestTime) { #ifndef ROCKSDB_LITE TEST_P(DBTestWithParam, MergeCompactionTimeTest) { SetPerfLevel(kEnableTime); + env_->skip_fsync_ = true; Options options = CurrentOptions(); options.compaction_filter_factory = std::make_shared(); options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); diff --git a/db/db_test_util.h b/db/db_test_util.h index 8b124b522..7a40a1346 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -276,7 +276,10 @@ class SpecialEnv : public EnvWrapper { while (env_->delay_sstable_sync_.load(std::memory_order_acquire)) { env_->SleepForMicroseconds(100000); } - Status s = base_->Sync(); + Status s; + if (!env_->skip_fsync_) { + s = base_->Sync(); + } #if !(defined NDEBUG) || !defined(OS_WIN) TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::Sync", &s); #endif // !(defined NDEBUG) || !defined(OS_WIN) @@ -314,7 +317,11 @@ class SpecialEnv : public EnvWrapper { if (env_->manifest_sync_error_.load(std::memory_order_acquire)) { return Status::IOError("simulated sync error"); } else { - return base_->Sync(); + if (env_->skip_fsync_) { + return Status::OK(); + } else { + return base_->Sync(); + } } } uint64_t GetFileSize() override { return base_->GetFileSize(); } @@ -369,7 +376,11 @@ class SpecialEnv : public EnvWrapper { Status Flush() override { return base_->Flush(); } Status Sync() override { ++env_->sync_counter_; - return base_->Sync(); + if (env_->skip_fsync_) { + return Status::OK(); + } else { + return base_->Sync(); + } } bool IsSyncThreadSafe() const override { return env_->is_wal_sync_thread_safe_.load(); @@ -382,6 +393,30 @@ class SpecialEnv : public EnvWrapper { SpecialEnv* env_; std::unique_ptr base_; }; + class OtherFile : public WritableFile { + public: + OtherFile(SpecialEnv* env, std::unique_ptr&& b) + : env_(env), base_(std::move(b)) {} + Status Append(const Slice& data) override { return base_->Append(data); } + Status Truncate(uint64_t size) override { return base_->Truncate(size); } + Status Close() override { return base_->Close(); } + Status Flush() override { return base_->Flush(); } + Status Sync() override { + if (env_->skip_fsync_) { + return Status::OK(); + } else { + return base_->Sync(); + } + } + uint64_t GetFileSize() override { return base_->GetFileSize(); } + Status Allocate(uint64_t offset, uint64_t len) override { + return base_->Allocate(offset, len); + } + + private: + SpecialEnv* env_; + std::unique_ptr base_; + }; if (non_writeable_rate_.load(std::memory_order_acquire) > 0) { uint32_t random_number; @@ -416,6 +451,8 @@ class SpecialEnv : public EnvWrapper { r->reset(new ManifestFile(this, std::move(*r))); } else if (strstr(f.c_str(), "log") != nullptr) { r->reset(new WalFile(this, std::move(*r))); + } else { + r->reset(new OtherFile(this, std::move(*r))); } } return s; @@ -546,6 +583,24 @@ class SpecialEnv : public EnvWrapper { options->stats_persist_period_sec = 0; } + Status NewDirectory(const std::string& name, + std::unique_ptr* result) override { + if (!skip_fsync_) { + return target()->NewDirectory(name, result); + } else { + class NoopDirectory : public Directory { + public: + NoopDirectory() {} + ~NoopDirectory() {} + + Status Fsync() override { return Status::OK(); } + }; + + result->reset(new NoopDirectory()); + return Status::OK(); + } + } + // Something to return when mocking current time const int64_t maybe_starting_time_; @@ -593,6 +648,9 @@ class SpecialEnv : public EnvWrapper { std::atomic sync_counter_; + // If true, all fsync to files and directories are skipped. + bool skip_fsync_ = false; + std::atomic non_writeable_rate_; std::atomic new_writable_count_; diff --git a/db/external_sst_file_test.cc b/db/external_sst_file_test.cc index f698d501e..2ae73c4c8 100644 --- a/db/external_sst_file_test.cc +++ b/db/external_sst_file_test.cc @@ -1123,6 +1123,7 @@ TEST_F(ExternalSSTFileTest, OverlappingRanges) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); do { Options options = CurrentOptions(); + env_->skip_fsync_ = true; DestroyAndReopen(options); SstFileWriter sst_file_writer(EnvOptions(), options); @@ -1733,6 +1734,7 @@ TEST_F(ExternalSSTFileTest, WithUnorderedWrite) { } TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoRandomized) { + env_->skip_fsync_ = true; Options options = CurrentOptions(); options.IncreaseParallelism(20); options.level0_slowdown_writes_trigger = 256;