Dynamically configure BlockBasedTableOptions.prepopulate_block_cache (#8620)

Summary:
Dynamically configure BlockBasedTableOptions.prepopulate_block_cache using DB::SetOptions.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8620

Test Plan: Added new unit test

Reviewed By: anand1976

Differential Revision: D30091319

Pulled By: akankshamahajan15

fbshipit-source-id: fb586d1848a8dd525bba7b2f9eeac34f2fc6d82c
main
Akanksha Mahajan 3 years ago committed by Facebook GitHub Bot
parent 9b25d26dc8
commit fd2079938d
  1. 1
      HISTORY.md
  2. 54
      db/db_block_cache_test.cc
  3. 4
      include/rocksdb/table.h
  4. 6
      include/rocksdb/utilities/options_type.h
  5. 3
      table/block_based/block_based_table_factory.cc

@ -10,6 +10,7 @@
* Made the EventListener extend the Customizable class. * Made the EventListener extend the Customizable class.
* EventListeners that have a non-empty Name() and that are registered with the ObjectRegistry can now be serialized to/from the OPTIONS file. * EventListeners that have a non-empty Name() and that are registered with the ObjectRegistry can now be serialized to/from the OPTIONS file.
* Insert warm blocks (data blocks, uncompressed dict blocks, index and filter blocks) in Block cache during flush under option BlockBasedTableOptions.prepopulate_block_cache. Previously it was enabled for only data blocks. * Insert warm blocks (data blocks, uncompressed dict blocks, index and filter blocks) in Block cache during flush under option BlockBasedTableOptions.prepopulate_block_cache. Previously it was enabled for only data blocks.
* BlockBasedTableOptions.prepopulate_block_cache can be dynamically configured using DB::SetOptions.
### Performance Improvements ### Performance Improvements
* Try to avoid updating DBOptions if `SetDBOptions()` does not change any option value. * Try to avoid updating DBOptions if `SetDBOptions()` does not change any option value.

@ -503,7 +503,7 @@ TEST_F(DBBlockCacheTest, WarmCacheWithDataBlocksDuringFlush) {
} }
} }
// This test cache all types of blocks during flush. // This test cache data, index and filter blocks during flush.
TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) { TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
@ -519,7 +519,7 @@ TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) {
DestroyAndReopen(options); DestroyAndReopen(options);
std::string value(kValueSize, 'a'); std::string value(kValueSize, 'a');
for (size_t i = 1; i < 2; i++) { for (size_t i = 1; i <= kNumBlocks; i++) {
ASSERT_OK(Put(ToString(i), value)); ASSERT_OK(Put(ToString(i), value));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD)); ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD));
@ -539,6 +539,56 @@ TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) {
options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT)); options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT));
} }
} }
TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) {
Options options = CurrentOptions();
options.create_if_missing = true;
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
BlockBasedTableOptions table_options;
table_options.block_cache = NewLRUCache(1 << 25, 0, false);
table_options.cache_index_and_filter_blocks = false;
table_options.prepopulate_block_cache =
BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options);
std::string value(kValueSize, 'a');
for (size_t i = 1; i <= 5; i++) {
ASSERT_OK(Put(ToString(i), value));
ASSERT_OK(Flush());
ASSERT_EQ(1,
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
ASSERT_EQ(value, Get(ToString(i)));
ASSERT_EQ(0,
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
ASSERT_EQ(
0, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
ASSERT_EQ(1,
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
}
ASSERT_OK(dbfull()->SetOptions(
{{"block_based_table_factory", "{prepopulate_block_cache=kDisable;}"}}));
for (size_t i = 6; i <= kNumBlocks; i++) {
ASSERT_OK(Put(ToString(i), value));
ASSERT_OK(Flush());
ASSERT_EQ(0,
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
ASSERT_EQ(value, Get(ToString(i)));
ASSERT_EQ(1,
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
ASSERT_EQ(
1, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS));
ASSERT_EQ(0,
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT));
}
}
#endif #endif
namespace { namespace {

@ -472,6 +472,10 @@ struct BlockBasedTableOptions {
// further helps if the workload exhibits high temporal locality, where most // further helps if the workload exhibits high temporal locality, where most
// of the reads go to recently written data. This also helps in case of // of the reads go to recently written data. This also helps in case of
// Distributed FileSystem. // Distributed FileSystem.
//
// This parameter can be changed dynamically by
// DB::SetOptions({{"block_based_table_factory",
// "{prepopulate_block_cache=kFlushOnly;}"}}));
enum class PrepopulateBlockCache : char { enum class PrepopulateBlockCache : char {
// Disable prepopulate block cache. // Disable prepopulate block cache.
kDisable, kDisable,

@ -262,10 +262,10 @@ class OptionTypeInfo {
// @param map The string to enum mapping for this enum // @param map The string to enum mapping for this enum
template <typename T> template <typename T>
static OptionTypeInfo Enum( static OptionTypeInfo Enum(
int offset, const std::unordered_map<std::string, T>* const map) { int offset, const std::unordered_map<std::string, T>* const map,
OptionTypeFlags flags = OptionTypeFlags::kNone) {
return OptionTypeInfo( return OptionTypeInfo(
offset, OptionType::kEnum, OptionVerificationType::kNormal, offset, OptionType::kEnum, OptionVerificationType::kNormal, flags,
OptionTypeFlags::kNone,
// Uses the map argument to convert the input string into // Uses the map argument to convert the input string into
// its corresponding enum value. If value is found in the map, // its corresponding enum value. If value is found in the map,
// addr is updated to the corresponding map entry. // addr is updated to the corresponding map entry.

@ -427,7 +427,8 @@ static std::unordered_map<std::string, OptionTypeInfo>
{"prepopulate_block_cache", {"prepopulate_block_cache",
OptionTypeInfo::Enum<BlockBasedTableOptions::PrepopulateBlockCache>( OptionTypeInfo::Enum<BlockBasedTableOptions::PrepopulateBlockCache>(
offsetof(struct BlockBasedTableOptions, prepopulate_block_cache), offsetof(struct BlockBasedTableOptions, prepopulate_block_cache),
&block_base_table_prepopulate_block_cache_string_map)}, &block_base_table_prepopulate_block_cache_string_map,
OptionTypeFlags::kMutable)},
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
}; };

Loading…
Cancel
Save