From 806d8916da26dfc82ed361a56dde790132f0d6a6 Mon Sep 17 00:00:00 2001 From: sdong Date: Tue, 14 Dec 2021 20:06:48 -0800 Subject: [PATCH] SimulatedHybridFileSystem to simulate HDD behavior more accurately (#9259) Summary: SimulatedHybridFileSystem now takes a more thorough simualtion of an HDD: 1. cover writes too, not just read 2. Latency and throughput is now simulated as seek + read time, using a rate limiter This implementation can be modified to simulate full HDD behavior, which is not yet done. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9259 Test Plan: Run db_bench and observe the desired behavior. Reviewed By: jay-zhuang Differential Revision: D32903039 fbshipit-source-id: a83f5d72143e114d5e75edf39d647bf0b71978e1 --- tools/simulated_hybrid_file_system.cc | 108 ++++++++++++++++++++++---- tools/simulated_hybrid_file_system.h | 29 +++++++ 2 files changed, 122 insertions(+), 15 deletions(-) diff --git a/tools/simulated_hybrid_file_system.cc b/tools/simulated_hybrid_file_system.cc index a53b8785d..776dc6623 100644 --- a/tools/simulated_hybrid_file_system.cc +++ b/tools/simulated_hybrid_file_system.cc @@ -16,8 +16,28 @@ namespace ROCKSDB_NAMESPACE { const int kLatencyAddedPerRequestUs = 15000; -const int64_t kRequestPerSec = 100; -const int64_t kDummyBytesPerRequest = 1024 * 1024; +const int64_t kUsPerSec = 1000000; +const int64_t kDummyBytesPerUs = 1024; + +namespace { +// From bytes to read/write, calculate service time needed by an HDD. +// This is used to simulate latency from HDD. +int CalculateServeTimeUs(size_t bytes) { + return 12200 + static_cast(static_cast(bytes) * 0.005215); +} + +// There is a bug in rater limiter that would crash with small requests +// Hack to get it around. +void RateLimiterRequest(RateLimiter* rater_limiter, int64_t amount) { + int64_t left = amount * kDummyBytesPerUs; + const int64_t kMaxToRequest = kDummyBytesPerUs * kUsPerSec / 1024; + while (left > 0) { + int64_t to_request = std::min(kMaxToRequest, left); + rater_limiter->Request(to_request, Env::IOPriority::IO_LOW, nullptr); + left -= to_request; + } +} +} // namespace // The metadata file format: each line is a full filename of a file which is // warm @@ -27,7 +47,7 @@ SimulatedHybridFileSystem::SimulatedHybridFileSystem( : FileSystemWrapper(base), // Limit to 100 requests per second. rate_limiter_(NewGenericRateLimiter( - kDummyBytesPerRequest * kRequestPerSec /* rate_bytes_per_sec */, + kDummyBytesPerUs * kUsPerSec /* rate_bytes_per_sec */, 1000 /* refill_period_us */)), metadata_file_name_(metadata_file_name), name_("SimulatedHybridFileSystem: " + std::string(target()->Name())) { @@ -91,10 +111,14 @@ IOStatus SimulatedHybridFileSystem::NewWritableFile( std::unique_ptr* result, IODebugContext* dbg) { if (file_opts.temperature == Temperature::kWarm) { const std::lock_guard lock(mutex_); - fprintf(stderr, "warm file %s\n", fname.c_str()); warm_file_set_.insert(fname); } - return target()->NewWritableFile(fname, file_opts, result, dbg); + + IOStatus s = target()->NewWritableFile(fname, file_opts, result, dbg); + if (file_opts.temperature == Temperature::kWarm) { + result->reset(new SimulatedWritableFile(std::move(*result), rate_limiter_)); + } + return s; } IOStatus SimulatedHybridFileSystem::DeleteFile(const std::string& fname, @@ -112,7 +136,7 @@ IOStatus SimulatedHybridRaf::Read(uint64_t offset, size_t n, char* scratch, IODebugContext* dbg) const { if (temperature_ == Temperature::kWarm) { Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs); - RequestRateLimit(1); + RequestRateLimit(n); } return target()->Read(offset, n, options, result, scratch, dbg); } @@ -121,7 +145,9 @@ IOStatus SimulatedHybridRaf::MultiRead(FSReadRequest* reqs, size_t num_reqs, const IOOptions& options, IODebugContext* dbg) { if (temperature_ == Temperature::kWarm) { - RequestRateLimit(static_cast(num_reqs)); + for (size_t i = 0; i < num_reqs; i++) { + RequestRateLimit(reqs[i].len); + } Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs * static_cast(num_reqs)); } @@ -132,22 +158,74 @@ IOStatus SimulatedHybridRaf::Prefetch(uint64_t offset, size_t n, const IOOptions& options, IODebugContext* dbg) { if (temperature_ == Temperature::kWarm) { - RequestRateLimit(1); + RequestRateLimit(n); Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs); } return target()->Prefetch(offset, n, options, dbg); } -void SimulatedHybridRaf::RequestRateLimit(int64_t num_requests) const { - int64_t left = num_requests * kDummyBytesPerRequest; - const int64_t kMaxToRequest = kDummyBytesPerRequest / 100; - while (left > 0) { - int64_t to_request = std::min(kMaxToRequest, left); - rate_limiter_->Request(to_request, Env::IOPriority::IO_LOW, nullptr); - left -= to_request; +void SimulatedHybridRaf::RequestRateLimit(int64_t bytes) const { + RateLimiterRequest(rate_limiter_.get(), CalculateServeTimeUs(bytes)); +} + +void SimulatedWritableFile::RequestRateLimit(int64_t bytes) const { + RateLimiterRequest(rate_limiter_.get(), CalculateServeTimeUs(bytes)); +} + +IOStatus SimulatedWritableFile::Append(const Slice& data, const IOOptions& ioo, + IODebugContext* idc) { + if (use_direct_io()) { + RequestRateLimit(data.size()); + } else { + unsynced_bytes += data.size(); + } + return target()->Append(data, ioo, idc); +} + +IOStatus SimulatedWritableFile::Append( + const Slice& data, const IOOptions& options, + const DataVerificationInfo& verification_info, IODebugContext* dbg) { + if (use_direct_io()) { + RequestRateLimit(data.size()); + } else { + unsynced_bytes += data.size(); } + return target()->Append(data, options, verification_info, dbg); } +IOStatus SimulatedWritableFile::PositionedAppend(const Slice& data, + uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) { + if (use_direct_io()) { + RequestRateLimit(data.size()); + } else { + // This might be overcalculated, but it's probably OK. + unsynced_bytes += data.size(); + } + return target()->PositionedAppend(data, offset, options, dbg); +} +IOStatus SimulatedWritableFile::PositionedAppend( + const Slice& data, uint64_t offset, const IOOptions& options, + const DataVerificationInfo& verification_info, IODebugContext* dbg) { + if (use_direct_io()) { + RequestRateLimit(data.size()); + } else { + // This might be overcalculated, but it's probably OK. + unsynced_bytes += data.size(); + } + return target()->PositionedAppend(data, offset, options, verification_info, + dbg); +} + +IOStatus SimulatedWritableFile::Sync(const IOOptions& options, + IODebugContext* dbg) { + if (unsynced_bytes > 0) { + RequestRateLimit(unsynced_bytes); + unsynced_bytes = 0; + } + return target()->Sync(options, dbg); +} } // namespace ROCKSDB_NAMESPACE #endif // ROCKSDB_LITE diff --git a/tools/simulated_hybrid_file_system.h b/tools/simulated_hybrid_file_system.h index 7b101b57a..e1b3393b4 100644 --- a/tools/simulated_hybrid_file_system.h +++ b/tools/simulated_hybrid_file_system.h @@ -86,6 +86,35 @@ class SimulatedHybridRaf : public FSRandomAccessFileOwnerWrapper { void RequestRateLimit(int64_t num_requests) const; }; + +class SimulatedWritableFile : public FSWritableFileWrapper { + public: + SimulatedWritableFile(std::unique_ptr&& t, + std::shared_ptr rate_limiter) + : FSWritableFileWrapper(t.get()), + file_guard_(std::move(t)), + rate_limiter_(rate_limiter) {} + IOStatus Append(const Slice& data, const IOOptions&, + IODebugContext*) override; + IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override; + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) override; + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override; + + private: + std::unique_ptr file_guard_; + std::shared_ptr rate_limiter_; + size_t unsynced_bytes = 0; + + void RequestRateLimit(int64_t num_requests) const; +}; } // namespace ROCKSDB_NAMESPACE #endif // ROCKSDB_LITE