From eeb27e1bbdedf5eb57f408035dc999847e1fd34a Mon Sep 17 00:00:00 2001 From: Kien-hung Li Date: Fri, 28 Oct 2016 13:36:05 -0400 Subject: [PATCH] Add handy option to turn on direct I/O in db_bench (#1424) --- include/rocksdb/options.h | 4 ++++ tools/db_bench_tool.cc | 6 ++++++ util/db_options.cc | 3 +++ util/db_options.h | 1 + util/env.cc | 1 + util/options.cc | 3 +++ util/options_helper.cc | 1 + util/options_helper.h | 3 +++ util/options_settable_test.cc | 1 + 9 files changed, 23 insertions(+) diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 70750c926..ff7ea12dd 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -1106,6 +1106,10 @@ struct DBOptions { // Default: false bool allow_mmap_writes; + // Use O_DIRECT for reading file + // Default: false + bool use_direct_reads; + // If false, fallocate() calls are bypassed bool allow_fallocate; diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index eb5a3b858..453c6dcae 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -761,6 +761,9 @@ DEFINE_bool(mmap_read, rocksdb::EnvOptions().use_mmap_reads, DEFINE_bool(mmap_write, rocksdb::EnvOptions().use_mmap_writes, "Allow writes to occur via mmap-ing files"); +DEFINE_bool(use_direct_reads, rocksdb::EnvOptions().use_direct_reads, + "Use O_DIRECT for reading data"); + DEFINE_bool(advise_random_on_open, rocksdb::Options().advise_random_on_open, "Advise random access on table file open"); @@ -2722,6 +2725,9 @@ class Benchmark { options.max_background_flushes = FLAGS_max_background_flushes; options.compaction_style = FLAGS_compaction_style_e; options.compaction_pri = FLAGS_compaction_pri_e; + options.allow_mmap_reads = FLAGS_mmap_read; + options.allow_mmap_writes = FLAGS_mmap_write; + options.use_direct_reads = FLAGS_use_direct_reads; if (FLAGS_prefix_size != 0) { options.prefix_extractor.reset( NewFixedPrefixTransform(FLAGS_prefix_size)); diff --git a/util/db_options.cc b/util/db_options.cc index c844b1f7c..e40840b68 100644 --- a/util/db_options.cc +++ b/util/db_options.cc @@ -56,6 +56,7 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) allow_os_buffer(options.allow_os_buffer), allow_mmap_reads(options.allow_mmap_reads), allow_mmap_writes(options.allow_mmap_writes), + use_direct_reads(options.use_direct_reads), allow_fallocate(options.allow_fallocate), is_fd_close_on_exec(options.is_fd_close_on_exec), stats_dump_period_sec(options.stats_dump_period_sec), @@ -132,6 +133,8 @@ void ImmutableDBOptions::Dump(Logger* log) const { allow_fallocate); Header(log, " Options.allow_mmap_writes: %d", allow_mmap_writes); + Header(log, " Options.use_direct_reads: %d", + use_direct_reads); Header(log, " Options.create_missing_column_families: %d", create_missing_column_families); Header(log, " Options.db_log_dir: %s", diff --git a/util/db_options.h b/util/db_options.h index e8fc90fd0..813e4f2cf 100644 --- a/util/db_options.h +++ b/util/db_options.h @@ -51,6 +51,7 @@ struct ImmutableDBOptions { bool allow_os_buffer; bool allow_mmap_reads; bool allow_mmap_writes; + bool use_direct_reads; bool allow_fallocate; bool is_fd_close_on_exec; unsigned int stats_dump_period_sec; diff --git a/util/env.cc b/util/env.cc index 38509c0f4..690377adb 100644 --- a/util/env.cc +++ b/util/env.cc @@ -316,6 +316,7 @@ void AssignEnvOptions(EnvOptions* env_options, const DBOptions& options) { env_options->use_os_buffer = options.allow_os_buffer; env_options->use_mmap_reads = options.allow_mmap_reads; env_options->use_mmap_writes = options.allow_mmap_writes; + env_options->use_direct_reads = options.use_direct_reads; env_options->set_fd_cloexec = options.is_fd_close_on_exec; env_options->bytes_per_sync = options.bytes_per_sync; env_options->compaction_readahead_size = options.compaction_readahead_size; diff --git a/util/options.cc b/util/options.cc index 36ea3275e..0423b7ab3 100644 --- a/util/options.cc +++ b/util/options.cc @@ -198,6 +198,7 @@ DBOptions::DBOptions() allow_os_buffer(true), allow_mmap_reads(false), allow_mmap_writes(false), + use_direct_reads(false), allow_fallocate(true), is_fd_close_on_exec(true), skip_log_error_on_recovery(false), @@ -267,6 +268,7 @@ DBOptions::DBOptions(const Options& options) allow_os_buffer(options.allow_os_buffer), allow_mmap_reads(options.allow_mmap_reads), allow_mmap_writes(options.allow_mmap_writes), + use_direct_reads(options.use_direct_reads), allow_fallocate(options.allow_fallocate), is_fd_close_on_exec(options.is_fd_close_on_exec), skip_log_error_on_recovery(options.skip_log_error_on_recovery), @@ -333,6 +335,7 @@ void DBOptions::Dump(Logger* log) const { Header(log, " Options.allow_mmap_reads: %d", allow_mmap_reads); Header(log, " Options.allow_fallocate: %d", allow_fallocate); Header(log, " Options.allow_mmap_writes: %d", allow_mmap_writes); + Header(log, " Options.use_direct_reads: %d", use_direct_reads); Header(log, " Options.create_missing_column_families: %d", create_missing_column_families); Header(log, " Options.db_log_dir: %s", diff --git a/util/options_helper.cc b/util/options_helper.cc index 9f80f3634..2818b64e4 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -74,6 +74,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.allow_os_buffer = immutable_db_options.allow_os_buffer; options.allow_mmap_reads = immutable_db_options.allow_mmap_reads; options.allow_mmap_writes = immutable_db_options.allow_mmap_writes; + options.use_direct_reads = immutable_db_options.use_direct_reads; options.allow_fallocate = immutable_db_options.allow_fallocate; options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec; options.stats_dump_period_sec = immutable_db_options.stats_dump_period_sec; diff --git a/util/options_helper.h b/util/options_helper.h index 037146a88..1e11ceb25 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -183,6 +183,9 @@ static std::unordered_map db_options_type_info = { {"allow_mmap_writes", {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"use_direct_reads", + {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, {"allow_2pc", {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, diff --git a/util/options_settable_test.cc b/util/options_settable_test.cc index 0afd2d85a..e56276953 100644 --- a/util/options_settable_test.cc +++ b/util/options_settable_test.cc @@ -274,6 +274,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { "stats_dump_period_sec=70127;" "allow_fallocate=true;" "allow_mmap_reads=false;" + "use_direct_reads=false;" "max_log_file_size=4607;" "random_access_max_buffer_size=1048576;" "advise_random_on_open=true;"