From 7a9ecdac3c13f2da361fb543ce8dd0371d49f4f4 Mon Sep 17 00:00:00 2001 From: Akanksha Mahajan Date: Fri, 9 Sep 2022 12:52:27 -0700 Subject: [PATCH] Add auto prefetching parameters to db_bench and db_stress (#10632) Summary: Same as title Pull Request resolved: https://github.com/facebook/rocksdb/pull/10632 Test Plan: make crash_test -j32 Reviewed By: anand1976 Differential Revision: D39241479 Pulled By: akankshamahajan15 fbshipit-source-id: 5db5b0c007da786bacc1b30d8926d36d6d029b87 --- db_stress_tool/db_stress_common.h | 5 +++++ db_stress_tool/db_stress_gflags.cc | 9 +++++++++ db_stress_tool/db_stress_test_base.cc | 6 ++++++ tools/db_bench_tool.cc | 10 ++++++++++ tools/db_crashtest.py | 4 ++++ 5 files changed, 34 insertions(+) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index a2e49429c..17a2072dd 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -314,6 +314,11 @@ DECLARE_int64(preclude_last_level_data_seconds); DECLARE_int32(verify_iterator_with_expected_state_one_in); +DECLARE_uint64(readahead_size); +DECLARE_uint64(initial_auto_readahead_size); +DECLARE_uint64(max_auto_readahead_size); +DECLARE_uint64(num_file_reads_for_auto_readahead); + constexpr long KB = 1024; constexpr int kRandomValueMaxFactor = 3; constexpr int kValueMaxLen = 100; diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 9af506fb9..654e022b6 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -1035,4 +1035,13 @@ DEFINE_int32(verify_iterator_with_expected_state_one_in, 0, "chance that the iterator is verified against the expected state " "file, instead of comparing keys between two iterators."); +DEFINE_uint64(readahead_size, 0, "Iterator readahead size"); +DEFINE_uint64(initial_auto_readahead_size, 0, + "Initial auto readahead size for prefetching during Iteration"); +DEFINE_uint64(max_auto_readahead_size, 0, + "Max auto readahead size for prefetching during Iteration"); +DEFINE_uint64( + num_file_reads_for_auto_readahead, 0, + "Num of sequential reads to enable auto prefetching during Iteration"); + #endif // GFLAGS diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 7955150f4..e2e0c05c1 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -635,6 +635,7 @@ void StressTest::OperateDb(ThreadState* thread) { FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; read_opts.async_io = FLAGS_async_io; read_opts.adaptive_readahead = FLAGS_adaptive_readahead; + read_opts.readahead_size = FLAGS_readahead_size; WriteOptions write_opts; if (FLAGS_rate_limit_auto_wal_flush) { write_opts.rate_limiter_priority = Env::IO_USER; @@ -2950,6 +2951,11 @@ void InitializeOptionsFromFlags( block_based_options.prepopulate_block_cache = static_cast( FLAGS_prepopulate_block_cache); + block_based_options.initial_auto_readahead_size = + FLAGS_initial_auto_readahead_size; + block_based_options.max_auto_readahead_size = FLAGS_max_auto_readahead_size; + block_based_options.num_file_reads_for_auto_readahead = + FLAGS_num_file_reads_for_auto_readahead; options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); options.db_write_buffer_size = FLAGS_db_write_buffer_size; options.write_buffer_size = FLAGS_write_buffer_size; diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 2c14779f7..5d662ab79 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -1236,6 +1236,14 @@ DEFINE_uint64( "BlockBasedTableOptions.initial_auto_readahead_size and doubles on every " "additional read upto max_auto_readahead_size"); +DEFINE_uint64( + num_file_reads_for_auto_readahead, + ROCKSDB_NAMESPACE::BlockBasedTableOptions() + .num_file_reads_for_auto_readahead, + "Rocksdb implicit readahead is enabled if reads are sequential and " + "num_file_reads_for_auto_readahead indicates after how many sequential " + "reads into that file internal auto prefetching should be start."); + static enum ROCKSDB_NAMESPACE::CompressionType StringToCompressionType( const char* ctype) { assert(ctype); @@ -4372,6 +4380,8 @@ class Benchmark { FLAGS_max_auto_readahead_size; block_based_options.initial_auto_readahead_size = FLAGS_initial_auto_readahead_size; + block_based_options.num_file_reads_for_auto_readahead = + FLAGS_num_file_reads_for_auto_readahead; BlockBasedTableOptions::PrepopulateBlockCache prepopulate_block_cache = block_based_options.prepopulate_block_cache; switch (FLAGS_prepopulate_block_cache) { diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 85a395eea..4d9ee74d4 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -180,6 +180,10 @@ default_params = { "secondary_cache_uri": lambda: random.choice( ["", "compressed_secondary_cache://capacity=8388608"]), "allow_data_in_errors": True, + "readahead_size": lambda: random.choice([0, 16384, 524288]), + "initial_auto_readahead_size": lambda: random.choice([0, 16384, 524288]), + "max_auto_readahead_size": lambda: random.choice([0, 16384, 524288]), + "num_file_reads_for_auto_readahead": lambda: random.choice([0, 1, 2]), } _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'