From e619a20e932bc382a2d2c03d14718d258f9cd46b Mon Sep 17 00:00:00 2001 From: Ziyue Yang Date: Thu, 30 Apr 2020 10:46:54 -0700 Subject: [PATCH] Add an option for parallel compression in for db_stress (#6722) Summary: This commit adds an `compression_parallel_threads` option in db_stress. It also fixes the naming of parallel compression option in db_bench to keep it aligned with others. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6722 Reviewed By: pdillinger Differential Revision: D21091385 fbshipit-source-id: c9ba8c4e5cc327ff9e6094a6dc6a15fcff70f100 --- db_stress_tool/db_stress_common.h | 1 + db_stress_tool/db_stress_gflags.cc | 3 +++ db_stress_tool/db_stress_test_base.cc | 2 ++ tools/db_bench_tool.cc | 7 ++++--- tools/db_crashtest.py | 1 + 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index ff71b889f..df3311a08 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -201,6 +201,7 @@ DECLARE_string(compression_type); DECLARE_string(bottommost_compression_type); DECLARE_int32(compression_max_dict_bytes); DECLARE_int32(compression_zstd_max_train_bytes); +DECLARE_int32(compression_parallel_threads); DECLARE_string(checksum_type); DECLARE_string(hdfs); DECLARE_string(env_uri); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index f4f538c66..9dd00ef4d 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -584,6 +584,9 @@ DEFINE_int32(compression_zstd_max_train_bytes, 0, "Maximum size of training data passed to zstd's dictionary " "trainer."); +DEFINE_int32(compression_parallel_threads, 1, + "Number of threads for parallel compression."); + DEFINE_string(bottommost_compression_type, "disable", "Algorithm to use to compress bottommost level of the database. " "\"disable\" means disabling the feature"); diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index bc0daed60..1ea3007d0 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -1803,6 +1803,8 @@ void StressTest::Open() { options_.compression_opts.max_dict_bytes = FLAGS_compression_max_dict_bytes; options_.compression_opts.zstd_max_train_bytes = FLAGS_compression_zstd_max_train_bytes; + options_.compression_opts.parallel_threads = + FLAGS_compression_parallel_threads; options_.create_if_missing = true; options_.max_manifest_file_size = FLAGS_max_manifest_file_size; options_.inplace_update_support = FLAGS_in_place_update; diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 52b907b09..65cbf6a83 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -926,8 +926,8 @@ DEFINE_int32(min_level_to_compress, -1, "If non-negative, compression starts" " not compressed. Otherwise, apply compression_type to " "all levels."); -DEFINE_int32(compression_threads, 1, - "Number of concurrent compression threads to run."); +DEFINE_int32(compression_parallel_threads, 1, + "Number of threads for parallel compression."); static bool ValidateTableCacheNumshardbits(const char* flagname, int32_t value) { @@ -4030,7 +4030,8 @@ class Benchmark { options.compression_opts.max_dict_bytes = FLAGS_compression_max_dict_bytes; options.compression_opts.zstd_max_train_bytes = FLAGS_compression_zstd_max_train_bytes; - options.compression_opts.parallel_threads = FLAGS_compression_threads; + options.compression_opts.parallel_threads = + FLAGS_compression_parallel_threads; // If this is a block based table, set some related options if (options.table_factory->Name() == BlockBasedTableFactory::kName && options.table_factory->GetOptions() != nullptr) { diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index e8cdb9d42..bb9e758b0 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -44,6 +44,7 @@ default_params = { "checksum_type" : lambda: random.choice(["kCRC32c", "kxxHash", "kxxHash64"]), "compression_max_dict_bytes": lambda: 16384 * random.randint(0, 1), "compression_zstd_max_train_bytes": lambda: 65536 * random.randint(0, 1), + "compression_parallel_threads": lambda: random.choice([1] * 9 + [4]), "clear_column_family_one_in": 0, "compact_files_one_in": 1000000, "compact_range_one_in": 1000000,