From fec7302a9d90f199561b34e7508731fee26f2f65 Mon Sep 17 00:00:00 2001 From: Maysam Yabandeh Date: Fri, 13 Dec 2019 10:23:01 -0800 Subject: [PATCH] Enable unordered_write in stress tests (#6164) Summary: With WritePrepared transactions configured with two_write_queues, unordered_write will offer the same guarantees as vanilla rocksdb and thus can be enabled in stress tests. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6164 Test Plan: ``` make -j32 crash_test_with_txn Differential Revision: D18991899 Pulled By: maysamyabandeh fbshipit-source-id: eece5e96b4169b67d7931e5c0afca88540a113e1 --- db_stress_tool/db_stress_common.h | 1 + db_stress_tool/db_stress_gflags.cc | 5 +++++ db_stress_tool/db_stress_test_base.cc | 10 ++++++++++ tools/db_crashtest.py | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index f4991a4a1..c02105a18 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -162,6 +162,7 @@ DECLARE_uint64(rate_limiter_bytes_per_sec); DECLARE_bool(rate_limit_bg_reads); DECLARE_bool(use_txn); DECLARE_uint64(txn_write_policy); +DECLARE_bool(unordered_write); DECLARE_int32(backup_one_in); DECLARE_int32(checkpoint_one_in); DECLARE_int32(ingest_external_file_one_in); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 5036be4d5..a6aff93c6 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -376,6 +376,11 @@ DEFINE_uint64(txn_write_policy, 0, "TxnDBWritePolicy::WRITE_COMMITTED. Note that this should not be " "changed accross crashes."); +DEFINE_bool(unordered_write, false, + "Turn on the unordered_write feature. This options is currently " + "tested only in combination with use_txn=true and " + "txn_write_policy=TxnDBWritePolicy::WRITE_PREPARED."); + DEFINE_int32(backup_one_in, 0, "If non-zero, then CreateNewBackup() will be called once for " "every N operations on average. 0 indicates CreateNewBackup() " diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index dd0e3b27a..390ea6750 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -1674,6 +1674,12 @@ void StressTest::Open() { assert(FLAGS_txn_write_policy <= TxnDBWritePolicy::WRITE_UNPREPARED); txn_db_options.write_policy = static_cast(FLAGS_txn_write_policy); + if (FLAGS_unordered_write) { + assert(txn_db_options.write_policy == TxnDBWritePolicy::WRITE_PREPARED); + options_.unordered_write = true; + options_.two_write_queues = true; + txn_db_options.skip_concurrency_control = true; + } s = TransactionDB::Open(options_, txn_db_options, FLAGS_db, cf_descriptors, &column_families_, &txn_db_); db_ = txn_db_; @@ -1773,6 +1779,10 @@ void StressTest::Reopen(ThreadState* thread) { #ifndef ROCKSDB_LITE if (thread->rand.OneIn(2)) { Status s = db_->Close(); + if (!s.ok()) { + fprintf(stderr, "Non-ok close status: %s\n", s.ToString().c_str()); + fflush(stderr); + } assert(s.ok()); } #endif diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index d96e02b8e..ed50b5e80 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -161,6 +161,7 @@ txn_params = { "use_txn" : 1, # Avoid lambda to set it once for the entire test "txn_write_policy": random.randint(0, 2), + "unordered_write": random.randint(0, 1), "disable_wal": 0, # OpenReadOnly after checkpoint is not currnetly compatible with WritePrepared txns "checkpoint_one_in": 0, @@ -185,6 +186,10 @@ def finalize_and_sanitize(src_params): dest_params.get("use_txn") == 1: dest_params["delpercent"] += dest_params["delrangepercent"] dest_params["delrangepercent"] = 0 + # Only under WritePrepared txns, unordered_write would provide the same guarnatees as vanilla rocksdb + if dest_params.get("unordered_write", 0) == 1: + dest_params["txn_write_policy"] = 1 + dest_params["allow_concurrent_memtable_write"] = 1 if dest_params.get("disable_wal", 0) == 1: dest_params["atomic_flush"] = 1 if dest_params.get("open_files", 1) != -1: