From fd2079938d784babcea908c6c82aeb8b3afa42af Mon Sep 17 00:00:00 2001 From: Akanksha Mahajan Date: Thu, 5 Aug 2021 19:43:44 -0700 Subject: [PATCH] Dynamically configure BlockBasedTableOptions.prepopulate_block_cache (#8620) Summary: Dynamically configure BlockBasedTableOptions.prepopulate_block_cache using DB::SetOptions. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8620 Test Plan: Added new unit test Reviewed By: anand1976 Differential Revision: D30091319 Pulled By: akankshamahajan15 fbshipit-source-id: fb586d1848a8dd525bba7b2f9eeac34f2fc6d82c --- HISTORY.md | 1 + db/db_block_cache_test.cc | 54 ++++++++++++++++++- include/rocksdb/table.h | 4 ++ include/rocksdb/utilities/options_type.h | 6 +-- .../block_based/block_based_table_factory.cc | 3 +- 5 files changed, 62 insertions(+), 6 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 17e48dc8f..eec1ad616 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -10,6 +10,7 @@ * Made the EventListener extend the Customizable class. * EventListeners that have a non-empty Name() and that are registered with the ObjectRegistry can now be serialized to/from the OPTIONS file. * Insert warm blocks (data blocks, uncompressed dict blocks, index and filter blocks) in Block cache during flush under option BlockBasedTableOptions.prepopulate_block_cache. Previously it was enabled for only data blocks. +* BlockBasedTableOptions.prepopulate_block_cache can be dynamically configured using DB::SetOptions. ### Performance Improvements * Try to avoid updating DBOptions if `SetDBOptions()` does not change any option value. diff --git a/db/db_block_cache_test.cc b/db/db_block_cache_test.cc index faf6f8a50..0b4389fd1 100644 --- a/db/db_block_cache_test.cc +++ b/db/db_block_cache_test.cc @@ -503,7 +503,7 @@ TEST_F(DBBlockCacheTest, WarmCacheWithDataBlocksDuringFlush) { } } -// This test cache all types of blocks during flush. +// This test cache data, index and filter blocks during flush. TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) { Options options = CurrentOptions(); options.create_if_missing = true; @@ -519,7 +519,7 @@ TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) { DestroyAndReopen(options); std::string value(kValueSize, 'a'); - for (size_t i = 1; i < 2; i++) { + for (size_t i = 1; i <= kNumBlocks; i++) { ASSERT_OK(Put(ToString(i), value)); ASSERT_OK(Flush()); ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD)); @@ -539,6 +539,56 @@ TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) { options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT)); } } + +TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(1 << 25, 0, false); + table_options.cache_index_and_filter_blocks = false; + table_options.prepopulate_block_cache = + BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly; + + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + std::string value(kValueSize, 'a'); + + for (size_t i = 1; i <= 5; i++) { + ASSERT_OK(Put(ToString(i), value)); + ASSERT_OK(Flush()); + ASSERT_EQ(1, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + + ASSERT_EQ(value, Get(ToString(i))); + ASSERT_EQ(0, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + ASSERT_EQ( + 0, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_EQ(1, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT)); + } + + ASSERT_OK(dbfull()->SetOptions( + {{"block_based_table_factory", "{prepopulate_block_cache=kDisable;}"}})); + + for (size_t i = 6; i <= kNumBlocks; i++) { + ASSERT_OK(Put(ToString(i), value)); + ASSERT_OK(Flush()); + ASSERT_EQ(0, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + + ASSERT_EQ(value, Get(ToString(i))); + ASSERT_EQ(1, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + ASSERT_EQ( + 1, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_EQ(0, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT)); + } +} #endif namespace { diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 94045ff6b..471102359 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -472,6 +472,10 @@ struct BlockBasedTableOptions { // further helps if the workload exhibits high temporal locality, where most // of the reads go to recently written data. This also helps in case of // Distributed FileSystem. + // + // This parameter can be changed dynamically by + // DB::SetOptions({{"block_based_table_factory", + // "{prepopulate_block_cache=kFlushOnly;}"}})); enum class PrepopulateBlockCache : char { // Disable prepopulate block cache. kDisable, diff --git a/include/rocksdb/utilities/options_type.h b/include/rocksdb/utilities/options_type.h index bf96ec732..a4e36752b 100644 --- a/include/rocksdb/utilities/options_type.h +++ b/include/rocksdb/utilities/options_type.h @@ -262,10 +262,10 @@ class OptionTypeInfo { // @param map The string to enum mapping for this enum template static OptionTypeInfo Enum( - int offset, const std::unordered_map* const map) { + int offset, const std::unordered_map* const map, + OptionTypeFlags flags = OptionTypeFlags::kNone) { return OptionTypeInfo( - offset, OptionType::kEnum, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, + offset, OptionType::kEnum, OptionVerificationType::kNormal, flags, // Uses the map argument to convert the input string into // its corresponding enum value. If value is found in the map, // addr is updated to the corresponding map entry. diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 75d177f18..db9d6f8d1 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -427,7 +427,8 @@ static std::unordered_map {"prepopulate_block_cache", OptionTypeInfo::Enum( offsetof(struct BlockBasedTableOptions, prepopulate_block_cache), - &block_base_table_prepopulate_block_cache_string_map)}, + &block_base_table_prepopulate_block_cache_string_map, + OptionTypeFlags::kMutable)}, #endif // ROCKSDB_LITE };