NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve SimulatedHybridFileSystem (#9301)

Browse Source 
Summary:
Several improvements to SimulatedHybridFileSystem:
(1) Allow a mode where all I/Os to all files simulate HDD. This can be enabled in db_bench using -simulate_hdd
(2) Latency calculation is slightly more accurate
(3) Allow to simulate more than one HDD spindles.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9301

Test Plan: Run db_bench and observe the results are reasonable.

Reviewed By: jay-zhuang

Differential Revision: D33141662

fbshipit-source-id: b736e58c4ba910d06899cc9ccec79b628275f4fa
main
sdong 3 years ago committed by Facebook GitHub Bot
parent 1c39b7952b
commit a931bacf5d
4 changed files with 58 additions and 29 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      HISTORY.md
  2. 11
      tools/db_bench_tool.cc
  3. 61
      tools/simulated_hybrid_file_system.cc
  4. 14
      tools/simulated_hybrid_file_system.h

1
HISTORY.md
Unescape View File

@ -7,6 +7,7 @@
## 6.28.0 (2021-12-17) ## 6.28.0 (2021-12-17)
### New Features ### New Features
* Introduced 'CommitWithTimestamp' as a new tag. Currently, there is no API for user to trigger a write with this tag to the WAL. This is part of the efforts to support write-commited transactions with user-defined timestamps. * Introduced 'CommitWithTimestamp' as a new tag. Currently, there is no API for user to trigger a write with this tag to the WAL. This is part of the efforts to support write-commited transactions with user-defined timestamps.
* Introduce SimulatedHybridFileSystem which can help simulating HDD latency in db_bench. Tiered Storage latency simulation can be enabled using -simulate_hybrid_fs_file (note that it doesn't work if db_bench is interrupted in the middle). -simulate_hdd can also be used to simulate all files on HDD.
### Bug Fixes ### Bug Fixes
* Fixed a bug in rocksdb automatic implicit prefetching which got broken because of new feature adaptive_readahead and internal prefetching got disabled when iterator moves from one file to next. * Fixed a bug in rocksdb automatic implicit prefetching which got broken because of new feature adaptive_readahead and internal prefetching got disabled when iterator moves from one file to next.

11
tools/db_bench_tool.cc
Unescape View File

@ -1154,6 +1154,10 @@ DEFINE_string(simulate_hybrid_fs_file, "",
"File for Store Metadata for Simulate hybrid FS. Empty means " "File for Store Metadata for Simulate hybrid FS. Empty means "
"disable the feature. Now, if it is set, " "disable the feature. Now, if it is set, "
"bottommost_temperature is set to kWarm."); "bottommost_temperature is set to kWarm.");
DEFINE_int32(simulate_hybrid_hdd_multipliers, 1,
"In simulate_hybrid_fs_file or simulate_hdd mode, how many HDDs "
"are simulated.");
DEFINE_bool(simulate_hdd, false, "Simulate read/write latency on HDD.");
static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_guard; static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_guard;
@ -8135,12 +8139,15 @@ int db_bench_tool(int argc, char** argv) {
fprintf(stderr, "Failed creating env: %s\n", s.ToString().c_str()); fprintf(stderr, "Failed creating env: %s\n", s.ToString().c_str());
exit(1); exit(1);
} }
} else if (FLAGS_simulate_hybrid_fs_file != "") { } else if (FLAGS_simulate_hdd || FLAGS_simulate_hybrid_fs_file != "") {
//**TODO: Make the simulate fs something that can be loaded //**TODO: Make the simulate fs something that can be loaded
// from the ObjectRegistry... // from the ObjectRegistry...
static std::shared_ptr<ROCKSDB_NAMESPACE::Env> composite_env = static std::shared_ptr<ROCKSDB_NAMESPACE::Env> composite_env =
NewCompositeEnv(std::make_shared<SimulatedHybridFileSystem>( NewCompositeEnv(std::make_shared<SimulatedHybridFileSystem>(
FileSystem::Default(), FLAGS_simulate_hybrid_fs_file)); FileSystem::Default(), FLAGS_simulate_hybrid_fs_file,
/*throughput_multiplier=*/
int{FLAGS_simulate_hybrid_hdd_multipliers},
/*is_full_fs_warm=*/FLAGS_simulate_hdd));
FLAGS_env = composite_env.get(); FLAGS_env = composite_env.get();
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE

61
tools/simulated_hybrid_file_system.cc
Unescape View File

@ -3,6 +3,7 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "util/stop_watch.h"
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include "tools/simulated_hybrid_file_system.h" #include "tools/simulated_hybrid_file_system.h"
@ -15,7 +16,6 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
const int kLatencyAddedPerRequestUs = 15000;
const int64_t kUsPerSec = 1000000; const int64_t kUsPerSec = 1000000;
const int64_t kDummyBytesPerUs = 1024; const int64_t kDummyBytesPerUs = 1024;
@ -43,14 +43,17 @@ void RateLimiterRequest(RateLimiter* rater_limiter, int64_t amount) {
// warm // warm
SimulatedHybridFileSystem::SimulatedHybridFileSystem( SimulatedHybridFileSystem::SimulatedHybridFileSystem(
const std::shared_ptr<FileSystem>& base, const std::shared_ptr<FileSystem>& base,
const std::string& metadata_file_name) const std::string& metadata_file_name, int throughput_multiplier,
bool is_full_fs_warm)
: FileSystemWrapper(base), : FileSystemWrapper(base),
// Limit to 100 requests per second. // Limit to 100 requests per second.
rate_limiter_(NewGenericRateLimiter( rate_limiter_(NewGenericRateLimiter(
kDummyBytesPerUs * kUsPerSec /* rate_bytes_per_sec */, int64_t{throughput_multiplier} * kDummyBytesPerUs *
kUsPerSec /* rate_bytes_per_sec */,
1000 /* refill_period_us */)), 1000 /* refill_period_us */)),
metadata_file_name_(metadata_file_name), metadata_file_name_(metadata_file_name),
name_("SimulatedHybridFileSystem: " + std::string(target()->Name())) { name_("SimulatedHybridFileSystem: " + std::string(target()->Name())),
is_full_fs_warm_(is_full_fs_warm) {
IOStatus s = base->FileExists(metadata_file_name, IOOptions(), nullptr); IOStatus s = base->FileExists(metadata_file_name, IOOptions(), nullptr);
if (s.IsNotFound()) { if (s.IsNotFound()) {
return; return;
@ -77,6 +80,9 @@ SimulatedHybridFileSystem::SimulatedHybridFileSystem(
// SimulatedHybridFileSystem::SimulatedHybridFileSystem() for format of the // SimulatedHybridFileSystem::SimulatedHybridFileSystem() for format of the
// file. // file.
SimulatedHybridFileSystem::~SimulatedHybridFileSystem() { SimulatedHybridFileSystem::~SimulatedHybridFileSystem() {
if (metadata_file_name_.empty()) {
return;
}
std::string metadata; std::string metadata;
for (const auto& f : warm_file_set_) { for (const auto& f : warm_file_set_) {
metadata += f; metadata += f;
@ -93,13 +99,15 @@ IOStatus SimulatedHybridFileSystem::NewRandomAccessFile(
const std::string& fname, const FileOptions& file_opts, const std::string& fname, const FileOptions& file_opts,
std::unique_ptr<FSRandomAccessFile>* result, IODebugContext* dbg) { std::unique_ptr<FSRandomAccessFile>* result, IODebugContext* dbg) {
Temperature temperature = Temperature::kUnknown; Temperature temperature = Temperature::kUnknown;
{ if (is_full_fs_warm_) {
temperature = Temperature::kWarm;
} else {
const std::lock_guard<std::mutex> lock(mutex_); const std::lock_guard<std::mutex> lock(mutex_);
if (warm_file_set_.find(fname) != warm_file_set_.end()) { if (warm_file_set_.find(fname) != warm_file_set_.end()) {
temperature = Temperature::kWarm; temperature = Temperature::kWarm;
} }
assert(temperature == file_opts.temperature);
} }
assert(temperature == file_opts.temperature);
IOStatus s = target()->NewRandomAccessFile(fname, file_opts, result, dbg); IOStatus s = target()->NewRandomAccessFile(fname, file_opts, result, dbg);
result->reset( result->reset(
new SimulatedHybridRaf(std::move(*result), rate_limiter_, temperature)); new SimulatedHybridRaf(std::move(*result), rate_limiter_, temperature));
@ -115,7 +123,7 @@ IOStatus SimulatedHybridFileSystem::NewWritableFile(
} }
IOStatus s = target()->NewWritableFile(fname, file_opts, result, dbg); IOStatus s = target()->NewWritableFile(fname, file_opts, result, dbg);
if (file_opts.temperature == Temperature::kWarm) { if (file_opts.temperature == Temperature::kWarm || is_full_fs_warm_) {
result->reset(new SimulatedWritableFile(std::move(*result), rate_limiter_)); result->reset(new SimulatedWritableFile(std::move(*result), rate_limiter_));
} }
return s; return s;
@ -135,8 +143,7 @@ IOStatus SimulatedHybridRaf::Read(uint64_t offset, size_t n,
const IOOptions& options, Slice* result, const IOOptions& options, Slice* result,
char* scratch, IODebugContext* dbg) const { char* scratch, IODebugContext* dbg) const {
if (temperature_ == Temperature::kWarm) { if (temperature_ == Temperature::kWarm) {
Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs); SimulateIOWait(n);
RequestRateLimit(n);
} }
return target()->Read(offset, n, options, result, scratch, dbg); return target()->Read(offset, n, options, result, scratch, dbg);
} }
@ -146,10 +153,8 @@ IOStatus SimulatedHybridRaf::MultiRead(FSReadRequest* reqs, size_t num_reqs,
IODebugContext* dbg) { IODebugContext* dbg) {
if (temperature_ == Temperature::kWarm) { if (temperature_ == Temperature::kWarm) {
for (size_t i = 0; i < num_reqs; i++) { for (size_t i = 0; i < num_reqs; i++) {
RequestRateLimit(reqs[i].len); SimulateIOWait(reqs[i].len);
} }
Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs *
static_cast<int>(num_reqs));
} }
return target()->MultiRead(reqs, num_reqs, options, dbg); return target()->MultiRead(reqs, num_reqs, options, dbg);
} }
@ -158,24 +163,34 @@ IOStatus SimulatedHybridRaf::Prefetch(uint64_t offset, size_t n,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
if (temperature_ == Temperature::kWarm) { if (temperature_ == Temperature::kWarm) {
RequestRateLimit(n); SimulateIOWait(n);
Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs);
} }
return target()->Prefetch(offset, n, options, dbg); return target()->Prefetch(offset, n, options, dbg);
} }
void SimulatedHybridRaf::RequestRateLimit(int64_t bytes) const { void SimulatedHybridRaf::SimulateIOWait(int64_t bytes) const {
RateLimiterRequest(rate_limiter_.get(), CalculateServeTimeUs(bytes)); int serve_time = CalculateServeTimeUs(bytes);
{
StopWatchNano stop_watch(Env::Default()->GetSystemClock().get(),
/*auto_start=*/true);
RateLimiterRequest(rate_limiter_.get(), serve_time);
int time_passed_us = static_cast<int>(stop_watch.ElapsedNanos() / 1000);
if (time_passed_us < serve_time) {
Env::Default()->SleepForMicroseconds(serve_time - time_passed_us);
}
}
} }
void SimulatedWritableFile::RequestRateLimit(int64_t bytes) const { void SimulatedWritableFile::SimulateIOWait(int64_t bytes) const {
RateLimiterRequest(rate_limiter_.get(), CalculateServeTimeUs(bytes)); int serve_time = CalculateServeTimeUs(bytes);
Env::Default()->SleepForMicroseconds(serve_time);
RateLimiterRequest(rate_limiter_.get(), serve_time);
} }
IOStatus SimulatedWritableFile::Append(const Slice& data, const IOOptions& ioo, IOStatus SimulatedWritableFile::Append(const Slice& data, const IOOptions& ioo,
IODebugContext* idc) { IODebugContext* idc) {
if (use_direct_io()) { if (use_direct_io()) {
RequestRateLimit(data.size()); SimulateIOWait(data.size());
} else { } else {
unsynced_bytes += data.size(); unsynced_bytes += data.size();
} }
@ -186,7 +201,7 @@ IOStatus SimulatedWritableFile::Append(
const Slice& data, const IOOptions& options, const Slice& data, const IOOptions& options,
const DataVerificationInfo& verification_info, IODebugContext* dbg) { const DataVerificationInfo& verification_info, IODebugContext* dbg) {
if (use_direct_io()) { if (use_direct_io()) {
RequestRateLimit(data.size()); SimulateIOWait(data.size());
} else { } else {
unsynced_bytes += data.size(); unsynced_bytes += data.size();
} }
@ -198,7 +213,7 @@ IOStatus SimulatedWritableFile::PositionedAppend(const Slice& data,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
if (use_direct_io()) { if (use_direct_io()) {
RequestRateLimit(data.size()); SimulateIOWait(data.size());
} else { } else {
// This might be overcalculated, but it's probably OK. // This might be overcalculated, but it's probably OK.
unsynced_bytes += data.size(); unsynced_bytes += data.size();
@ -209,7 +224,7 @@ IOStatus SimulatedWritableFile::PositionedAppend(
const Slice& data, uint64_t offset, const IOOptions& options, const Slice& data, uint64_t offset, const IOOptions& options,
const DataVerificationInfo& verification_info, IODebugContext* dbg) { const DataVerificationInfo& verification_info, IODebugContext* dbg) {
if (use_direct_io()) { if (use_direct_io()) {
RequestRateLimit(data.size()); SimulateIOWait(data.size());
} else { } else {
// This might be overcalculated, but it's probably OK. // This might be overcalculated, but it's probably OK.
unsynced_bytes += data.size(); unsynced_bytes += data.size();
@ -221,7 +236,7 @@ IOStatus SimulatedWritableFile::PositionedAppend(
IOStatus SimulatedWritableFile::Sync(const IOOptions& options, IOStatus SimulatedWritableFile::Sync(const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
if (unsynced_bytes > 0) { if (unsynced_bytes > 0) {
RequestRateLimit(unsynced_bytes); SimulateIOWait(unsynced_bytes);
unsynced_bytes = 0; unsynced_bytes = 0;
} }
return target()->Sync(options, dbg); return target()->Sync(options, dbg);

14
tools/simulated_hybrid_file_system.h
Unescape View File

@ -28,8 +28,13 @@ class SimulatedHybridFileSystem : public FileSystemWrapper {
// metadata_file_name stores metadata of the files, so that it can be // metadata_file_name stores metadata of the files, so that it can be
// loaded after process restarts. If the file doesn't exist, create // loaded after process restarts. If the file doesn't exist, create
// one. The file is written when the class is destroyed. // one. The file is written when the class is destroyed.
explicit SimulatedHybridFileSystem(const std::shared_ptr<FileSystem>& base, // throughput_multiplier: multiplier of throughput. For example, 1 is to
const std::string& metadata_file_name); // simulate single disk spindle. 4 is to simualte 4 disk spindles.
// is_full_fs_warm: if true, all files are all included in slow I/O
// simulation.
SimulatedHybridFileSystem(const std::shared_ptr<FileSystem>& base,
const std::string& metadata_file_name,
int throughput_multiplier, bool is_full_fs_warm);
~SimulatedHybridFileSystem() override; ~SimulatedHybridFileSystem() override;
@ -55,6 +60,7 @@ class SimulatedHybridFileSystem : public FileSystemWrapper {
std::unordered_set<std::string> warm_file_set_; std::unordered_set<std::string> warm_file_set_;
std::string metadata_file_name_; std::string metadata_file_name_;
std::string name_; std::string name_;
bool is_full_fs_warm_;
}; };
// Simulated random access file that can control IOPs and latency to simulate // Simulated random access file that can control IOPs and latency to simulate
@ -84,7 +90,7 @@ class SimulatedHybridRaf : public FSRandomAccessFileOwnerWrapper {
std::shared_ptr<RateLimiter> rate_limiter_; std::shared_ptr<RateLimiter> rate_limiter_;
Temperature temperature_; Temperature temperature_;
void RequestRateLimit(int64_t num_requests) const; void SimulateIOWait(int64_t num_requests) const;
}; };
class SimulatedWritableFile : public FSWritableFileWrapper { class SimulatedWritableFile : public FSWritableFileWrapper {
@ -113,7 +119,7 @@ class SimulatedWritableFile : public FSWritableFileWrapper {
std::shared_ptr<RateLimiter> rate_limiter_; std::shared_ptr<RateLimiter> rate_limiter_;
size_t unsynced_bytes = 0; size_t unsynced_bytes = 0;
void RequestRateLimit(int64_t num_requests) const; void SimulateIOWait(int64_t num_requests) const;
}; };
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

Write Preview
Loading…
Cancel
Save