From b4d0e041d05b445a914a56bfe5459ebce77486c6 Mon Sep 17 00:00:00 2001 From: Guido Tagliavini Ponce Date: Wed, 1 Jun 2022 18:00:28 -0700 Subject: [PATCH] Add support for FastLRUCache in stress and crash tests. (#10081) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Stress tests can run with the experimental FastLRUCache. Crash tests randomly choose between LRUCache and FastLRUCache. Since only LRUCache supports a secondary cache, we validate the `--secondary_cache_uri` and `--cache_type` flags---when `--secondary_cache_uri` is set, the `--cache_type` is set to `lru_cache`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10081 Test Plan: - To test that the FastLRUCache is used and the stress test runs successfully, run `make -j24 CRASH_TEST_EXT_ARGS=—duration=960 blackbox_crash_test_with_atomic_flush`. The cache type should sometimes be `fast_lru_cache`. - To test the flag validation, run `make -j24 CRASH_TEST_EXT_ARGS="--duration=960 --secondary_cache_uri=x" blackbox_crash_test_with_atomic_flush` multiple times. The test will always be aborted (which is okay). Check that the cache type is always `lru_cache`. Reviewed By: anand1976 Differential Revision: D36839908 Pulled By: guidotag fbshipit-source-id: ebcdfdcd12ec04c96c09ae5b9c9d1e613bdd1725 --- db_stress_tool/db_stress_common.h | 2 +- db_stress_tool/db_stress_gflags.cc | 3 +-- db_stress_tool/db_stress_test_base.cc | 11 +++++++++-- tools/db_crashtest.py | 16 ++++++++++------ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index c1d75d137..033ecd43b 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -142,7 +142,7 @@ DECLARE_bool(charge_table_reader); DECLARE_int32(top_level_index_pinning); DECLARE_int32(partition_pinning); DECLARE_int32(unpartitioned_pinning); -DECLARE_bool(use_clock_cache); +DECLARE_string(cache_type); DECLARE_uint64(subcompactions); DECLARE_uint64(periodic_compaction_seconds); DECLARE_uint64(compaction_ttl); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index eb1a5b0c4..b48fbf429 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -343,8 +343,7 @@ DEFINE_int32( "Type of pinning for unpartitioned metadata blocks (see `enum PinningTier` " "in table.h)"); -DEFINE_bool(use_clock_cache, false, - "Replace default LRU block cache with clock cache."); +DEFINE_string(cache_type, "lru_cache", "Type of block cache."); DEFINE_uint64(subcompactions, 1, "Maximum number of subcompactions to divide L0-L1 compactions " diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 0c8b7ee08..8b80f4d21 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -10,6 +10,7 @@ #include "util/compression.h" #ifdef GFLAGS +#include "cache/fast_lru_cache.h" #include "db_stress_tool/db_stress_common.h" #include "db_stress_tool/db_stress_compaction_filter.h" #include "db_stress_tool/db_stress_driver.h" @@ -130,14 +131,17 @@ std::shared_ptr StressTest::NewCache(size_t capacity, if (capacity <= 0) { return nullptr; } - if (FLAGS_use_clock_cache) { + + if (FLAGS_cache_type == "clock_cache") { auto cache = NewClockCache((size_t)capacity); if (!cache) { fprintf(stderr, "Clock cache not supported."); exit(1); } return cache; - } else { + } else if (FLAGS_cache_type == "fast_lru_cache") { + return NewFastLRUCache((size_t)capacity, num_shard_bits); + } else if (FLAGS_cache_type == "lru_cache") { LRUCacheOptions opts; opts.capacity = capacity; opts.num_shard_bits = num_shard_bits; @@ -161,6 +165,9 @@ std::shared_ptr StressTest::NewCache(size_t capacity, } #endif return NewLRUCache(opts); + } else { + fprintf(stderr, "Cache type not supported."); + exit(1); } } diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 005cda1ce..bf0f5655d 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -113,7 +113,7 @@ default_params = { "use_direct_reads": lambda: random.randint(0, 1), "use_direct_io_for_flush_and_compaction": lambda: random.randint(0, 1), "mock_direct_io": False, - "use_clock_cache": 0, # currently broken + "cache_type": lambda: random.choice(["fast_lru_cache", "lru_cache"]), # clock_cache is broken "use_full_merge_v1": lambda: random.randint(0, 1), "use_merge": lambda: random.randint(0, 1), # 999 -> use Bloom API @@ -176,6 +176,7 @@ default_params = { "async_io": lambda: random.choice([0, 1]), "wal_compression": lambda: random.choice(["none", "zstd"]), "verify_sst_unique_id_in_manifest": 1, # always do unique_id verification + "secondary_cache_uri": "", } _TEST_DIR_ENV_VAR = 'TEST_TMPDIR' @@ -525,11 +526,14 @@ def finalize_and_sanitize(src_params): if dest_params.get("two_write_queues") == 1: dest_params["enable_pipelined_write"] = 0 if dest_params.get("best_efforts_recovery") == 1: - dest_params["disable_wal"] = 1 - dest_params["atomic_flush"] = 0 - dest_params["enable_compaction_filter"] = 0 - dest_params["sync"] = 0 - dest_params["write_fault_one_in"] = 0 + dest_params["disable_wal"] = 1 + dest_params["atomic_flush"] = 0 + dest_params["enable_compaction_filter"] = 0 + dest_params["sync"] = 0 + dest_params["write_fault_one_in"] = 0 + if dest_params["secondary_cache_uri"] != "": + # Currently the only cache type compatible with a secondary cache is LRUCache + dest_params["cache_type"] = "lru_cache" return dest_params