From a11df583ecd68bab43c69f4436f5a57829ceead3 Mon Sep 17 00:00:00 2001 From: Fenggang Wu Date: Fri, 27 Jul 2018 15:35:41 -0700 Subject: [PATCH] Add DataBlockIndexType option in BlockBasedTableOptions (#4150) Summary: Added DataBlockIndexType option in BlockBasedTableOptions. ``` enum DataBlockIndexType : char { kDataBlockBinarySearch = 0, // traditional block type kDataBlockHashIndex = 1, // additional hash index appended to the end. }; ``` The default type is the traditional binary seek option: `kDataBlockBinarySearch`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4150 Differential Revision: D8895958 Pulled By: fgwu fbshipit-source-id: 480adef48104cf11d30db3bad9a73f98b4a80c10 --- include/rocksdb/table.h | 9 +++++++++ options/options.cc | 2 ++ options/options_helper.cc | 18 ++++++++++++++++++ options/options_helper.h | 6 ++++++ options/options_parser.cc | 6 ++++++ options/options_settable_test.cc | 1 + table/block_based_table_factory.h | 4 ++++ 7 files changed, 46 insertions(+) diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 413a92a49..64275339c 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -100,6 +100,15 @@ struct BlockBasedTableOptions { IndexType index_type = kBinarySearch; + // The index type that will be used for the data block. + // Now two DataBlockIndexType supported + enum DataBlockIndexType : char { + kDataBlockBinarySearch = 0, // traditional block type + kDataBlockHashSearch = 1, // additional hash index appended to the end. + }; + + DataBlockIndexType data_block_index_type = kDataBlockBinarySearch; + // This option is now deprecated. No matter what value it is set to, // it will behave as if hash_index_allow_collision=true. bool hash_index_allow_collision = true; diff --git a/options/options.cc b/options/options.cc index 49c84c46a..5527669d3 100644 --- a/options/options.cc +++ b/options/options.cc @@ -479,6 +479,8 @@ ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup( prefix_extractor.reset(NewNoopTransform()); BlockBasedTableOptions block_based_options; block_based_options.index_type = BlockBasedTableOptions::kHashSearch; + block_based_options.data_block_index_type = + BlockBasedTableOptions::kDataBlockBinarySearch; block_based_options.filter_policy.reset(NewBloomFilterPolicy(10)); block_based_options.block_cache = NewLRUCache(static_cast(block_cache_size_mb * 1024 * 1024)); diff --git a/options/options_helper.cc b/options/options_helper.cc index 455b53503..de481ad64 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -494,6 +494,11 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, return ParseEnum( block_base_table_index_type_string_map, value, reinterpret_cast(opt_address)); + case OptionType::kBlockBasedTableDataBlockIndexType: + return ParseEnum( + block_base_table_data_block_index_type_string_map, value, + reinterpret_cast( + opt_address)); case OptionType::kEncodingType: return ParseEnum( encoding_type_string_map, value, @@ -673,6 +678,12 @@ bool SerializeSingleOptionHelper(const char* opt_address, *reinterpret_cast( opt_address), value); + case OptionType::kBlockBasedTableDataBlockIndexType: + return SerializeEnum( + block_base_table_data_block_index_type_string_map, + *reinterpret_cast( + opt_address), + value); case OptionType::kFlushBlockPolicyFactory: { const auto* ptr = reinterpret_cast*>( @@ -1552,6 +1563,13 @@ std::unordered_map {"kTwoLevelIndexSearch", BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch}}; +std::unordered_map + OptionsHelper::block_base_table_data_block_index_type_string_map = { + {"kDataBlockBinarySearch", + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch}, + {"kDataBlockHashSearch", + BlockBasedTableOptions::DataBlockIndexType::kDataBlockHashSearch}}; + std::unordered_map OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain}, {"kPrefix", kPrefix}}; diff --git a/options/options_helper.h b/options/options_helper.h index 2ec22077a..016a0a1af 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -69,6 +69,7 @@ enum class OptionType { kMergeOperator, kMemTableRepFactory, kBlockBasedTableIndexType, + kBlockBasedTableDataBlockIndexType, kFilterPolicy, kFlushBlockPolicyFactory, kChecksumType, @@ -163,6 +164,9 @@ struct OptionsHelper { lru_cache_options_type_info; static std::unordered_map block_base_table_index_type_string_map; + static std::unordered_map + block_base_table_data_block_index_type_string_map; static std::unordered_map encoding_type_string_map; static std::unordered_map compaction_style_string_map; @@ -203,6 +207,8 @@ static auto& compression_type_string_map = OptionsHelper::compression_type_string_map; static auto& block_base_table_index_type_string_map = OptionsHelper::block_base_table_index_type_string_map; +static auto& block_base_table_data_block_index_type_string_map = + OptionsHelper::block_base_table_data_block_index_type_string_map; static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map; static auto& compaction_style_string_map = OptionsHelper::compaction_style_string_map; diff --git a/options/options_parser.cc b/options/options_parser.cc index dabefdc7b..01d8ffcb8 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -592,6 +592,12 @@ bool AreEqualOptions( *reinterpret_cast( offset1) == *reinterpret_cast(offset2)); + case OptionType::kBlockBasedTableDataBlockIndexType: + return ( + *reinterpret_cast( + offset1) == + *reinterpret_cast( + offset2)); case OptionType::kWALRecoveryMode: return (*reinterpret_cast(offset1) == *reinterpret_cast(offset2)); diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index eabfea8c8..9a154d92c 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -142,6 +142,7 @@ TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) { "pin_l0_filter_and_index_blocks_in_cache=1;" "pin_top_level_index_and_filter=1;" "index_type=kHashSearch;" + "data_block_index_type=kDataBlockHashSearch;" "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" "block_cache=1M;block_cache_compressed=1k;block_size=1024;" "block_size_deviation=8;block_restart_interval=4; " diff --git a/table/block_based_table_factory.h b/table/block_based_table_factory.h index 6f05d6040..b0368328f 100644 --- a/table/block_based_table_factory.h +++ b/table/block_based_table_factory.h @@ -123,6 +123,10 @@ static std::unordered_map {"hash_index_allow_collision", {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"data_block_index_type", + {offsetof(struct BlockBasedTableOptions, data_block_index_type), + OptionType::kBlockBasedTableDataBlockIndexType, + OptionVerificationType::kNormal, false, 0}}, {"checksum", {offsetof(struct BlockBasedTableOptions, checksum), OptionType::kChecksumType, OptionVerificationType::kNormal, false,