From 9d646a6311c4940bc69b8b95f073abf938eff70d Mon Sep 17 00:00:00 2001 From: Fenggang Wu Date: Thu, 16 Aug 2018 18:29:13 -0700 Subject: [PATCH] Add db_bench options of data block hash index (#4281) Summary: Add `--data_block_index_type` and `--data_block_hash_table_util_ratio` option to `db_bench`. `--data_block_index_type` can be either of `binary` (default) or `binary_and_hash`; `--data_block_hash_table_util_ratio` will be a double. The default value is `0.75`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4281 Differential Revision: D9361476 Pulled By: fgwu fbshipit-source-id: dc53e01acef9db81b9eec5e8a96f3bc8ed718c10 --- include/rocksdb/table.h | 5 ++--- table/block.cc | 10 +++++----- table/block_based_table_factory.cc | 2 +- table/block_builder.cc | 7 +++---- table/data_block_footer.cc | 5 ++--- table/data_block_footer.h | 3 +-- table/data_block_hash_index.cc | 8 ++++---- table/data_block_hash_index_test.cc | 26 ++++++++++++-------------- table/table_test.cc | 16 +++++++--------- tools/db_bench_tool.cc | 19 +++++++++++++++++++ 10 files changed, 56 insertions(+), 45 deletions(-) diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 412bf3883..701fff8b4 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -102,8 +102,8 @@ struct BlockBasedTableOptions { // The index type that will be used for the data block. enum DataBlockIndexType : char { - kDataBlockBinarySearch = 0, // traditional block type - kDataBlockBinaryAndHash = 1, // additional hash index + kDataBlockBinarySearch = 0, // traditional block type + kDataBlockBinaryAndHash = 1, // additional hash index }; DataBlockIndexType data_block_index_type = kDataBlockBinarySearch; @@ -112,7 +112,6 @@ struct BlockBasedTableOptions { // kDataBlockBinaryAndHash. double data_block_hash_table_util_ratio = 0.75; - // This option is now deprecated. No matter what value it is set to, // it will behave as if hash_index_allow_collision=true. bool hash_index_allow_collision = true; diff --git a/table/block.cc b/table/block.cc index 39c5e90ec..f26416e1b 100644 --- a/table/block.cc +++ b/table/block.cc @@ -262,7 +262,7 @@ bool DataBlockIter::SeekForGetImpl(const Slice& target) { // Even if the user_key is not found in the hash map, the caller still // have to conntinue searching the next block. So we invalidate the // iterator to tell the caller to go on. - current_ = restarts_; // Invalidate the iter + current_ = restarts_; // Invalidate the iter return true; } @@ -781,7 +781,7 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno, break; case BlockBasedTableOptions::kDataBlockBinaryAndHash: if (size_ < sizeof(uint32_t) /* block footer */ + - sizeof(uint16_t) /* NUM_BUCK */) { + sizeof(uint16_t) /* NUM_BUCK */) { size_ = 0; break; } @@ -789,9 +789,9 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno, uint16_t map_offset; data_block_hash_index_.Initialize( contents.data.data(), - static_cast( - contents.data.size() - sizeof(uint32_t)), /*chop off - NUM_RESTARTS*/ + static_cast(contents.data.size() - + sizeof(uint32_t)), /*chop off + NUM_RESTARTS*/ &map_offset); restart_offset_ = map_offset - num_restarts_ * sizeof(uint32_t); diff --git a/table/block_based_table_factory.cc b/table/block_based_table_factory.cc index 831619936..485aed870 100644 --- a/table/block_based_table_factory.cc +++ b/table/block_based_table_factory.cc @@ -260,7 +260,7 @@ Status BlockBasedTableFactory::SanitizeOptions( "Block alignment requested but block size is not a power of 2"); } if (table_options_.data_block_index_type == - BlockBasedTableOptions::kDataBlockBinaryAndHash && + BlockBasedTableOptions::kDataBlockBinaryAndHash && table_options_.data_block_hash_table_util_ratio <= 0) { return Status::InvalidArgument( "data_block_hash_table_util_ratio should be greater than 0 when " diff --git a/table/block_builder.cc b/table/block_builder.cc index c33007c7a..d7051d2db 100644 --- a/table/block_builder.cc +++ b/table/block_builder.cc @@ -33,8 +33,8 @@ #include "table/block_builder.h" -#include #include +#include #include "db/dbformat.h" #include "rocksdb/comparator.h" #include "table/data_block_footer.h" @@ -118,7 +118,7 @@ Slice BlockBuilder::Finish() { uint32_t num_restarts = static_cast(restarts_.size()); BlockBasedTableOptions::DataBlockIndexType index_type = - BlockBasedTableOptions::kDataBlockBinarySearch; + BlockBasedTableOptions::kDataBlockBinarySearch; if (data_block_hash_index_builder_.Valid() && CurrentSizeEstimate() <= kMaxBlockSizeSupportedByHashIndex) { data_block_hash_index_builder_.Finish(buffer_); @@ -126,8 +126,7 @@ Slice BlockBuilder::Finish() { } // footer is a packed format of data_block_index_type and num_restarts - uint32_t block_footer = PackIndexTypeAndNumRestarts( - index_type, num_restarts); + uint32_t block_footer = PackIndexTypeAndNumRestarts(index_type, num_restarts); PutFixed32(&buffer_, block_footer); finished_ = true; diff --git a/table/data_block_footer.cc b/table/data_block_footer.cc index dab5c4af6..cb9e14381 100644 --- a/table/data_block_footer.cc +++ b/table/data_block_footer.cc @@ -32,13 +32,12 @@ uint32_t PackIndexTypeAndNumRestarts( if (index_type == BlockBasedTableOptions::kDataBlockBinaryAndHash) { block_footer |= 1u << kDataBlockIndexTypeBitShift; } else if (index_type != BlockBasedTableOptions::kDataBlockBinarySearch) { - assert(0); + assert(0); } return block_footer; } - void UnPackIndexTypeAndNumRestarts( uint32_t block_footer, BlockBasedTableOptions::DataBlockIndexType* index_type, @@ -57,4 +56,4 @@ void UnPackIndexTypeAndNumRestarts( } } -} // namespace rocksdb +} // namespace rocksdb diff --git a/table/data_block_footer.h b/table/data_block_footer.h index 2757697a7..e6ff20bcc 100644 --- a/table/data_block_footer.h +++ b/table/data_block_footer.h @@ -17,10 +17,9 @@ uint32_t PackIndexTypeAndNumRestarts( BlockBasedTableOptions::DataBlockIndexType index_type, uint32_t num_restarts); - void UnPackIndexTypeAndNumRestarts( uint32_t block_footer, BlockBasedTableOptions::DataBlockIndexType* index_type, uint32_t* num_restarts); -} // namespace rocksdb +} // namespace rocksdb diff --git a/table/data_block_hash_index.cc b/table/data_block_hash_index.cc index c4c3dfb87..bdb784622 100644 --- a/table/data_block_hash_index.cc +++ b/table/data_block_hash_index.cc @@ -54,9 +54,9 @@ void DataBlockHashIndexBuilder::Finish(std::string& buffer) { } for (uint8_t restart_index : buckets) { - buffer.append(const_cast( - reinterpret_cast(&restart_index)), - sizeof(restart_index)); + buffer.append( + const_cast(reinterpret_cast(&restart_index)), + sizeof(restart_index)); } // write NUM_BUCK @@ -81,7 +81,7 @@ void DataBlockHashIndex::Initialize(const char* data, uint16_t size, } uint8_t DataBlockHashIndex::Lookup(const char* data, uint32_t map_offset, - const Slice& key) const { + const Slice& key) const { uint32_t hash_value = GetSliceHash(key); uint16_t idx = static_cast(hash_value % num_buckets_); const char* bucket_table = data + map_offset; diff --git a/table/data_block_hash_index_test.cc b/table/data_block_hash_index_test.cc index 3d112422f..c2b059893 100644 --- a/table/data_block_hash_index_test.cc +++ b/table/data_block_hash_index_test.cc @@ -9,10 +9,10 @@ #include "rocksdb/slice.h" #include "table/block.h" +#include "table/block_based_table_reader.h" #include "table/block_builder.h" #include "table/data_block_hash_index.h" #include "table/get_context.h" -#include "table/block_based_table_reader.h" #include "util/testharness.h" #include "util/testutil.h" @@ -40,9 +40,9 @@ static std::string RandomString(Random* rnd, int len) { return r; } std::string GenerateKey(int primary_key, int secondary_key, int padding_size, - Random *rnd) { + Random* rnd) { char buf[50]; - char *p = &buf[0]; + char* p = &buf[0]; snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key); std::string k(p); if (padding_size) { @@ -55,8 +55,8 @@ std::string GenerateKey(int primary_key, int secondary_key, int padding_size, // Generate random key value pairs. // The generated key will be sorted. You can tune the parameters to generated // different kinds of test key/value pairs for different scenario. -void GenerateRandomKVs(std::vector *keys, - std::vector *values, const int from, +void GenerateRandomKVs(std::vector* keys, + std::vector* values, const int from, const int len, const int step = 1, const int padding_size = 0, const int keys_share_prefix = 1) { @@ -93,8 +93,7 @@ TEST(DataBlockHashIndex, DataBlockHashTestSmall) { ASSERT_EQ(buffer.size(), estimated_size); - buffer2 = buffer; // test for the correctness of relative offset - + buffer2 = buffer; // test for the correctness of relative offset Slice s(buffer2); DataBlockHashIndex index; @@ -290,7 +289,6 @@ TEST(DataBlockHashIndex, BlockRestartIndexExceedMax) { BlockBasedTableOptions::kDataBlockBinaryAndHash); } - builder.Reset(); // #restarts > 253. HashIndex is not used @@ -343,7 +341,7 @@ TEST(DataBlockHashIndex, BlockSizeExceedMax) { Block reader(std::move(contents), kDisableGlobalSequenceNumber); ASSERT_EQ(reader.IndexType(), - BlockBasedTableOptions::kDataBlockBinaryAndHash); + BlockBasedTableOptions::kDataBlockBinaryAndHash); } builder.Reset(); @@ -404,8 +402,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) { may_exist = iter->SeekForGet(seek_ikey.Encode().ToString()); ASSERT_TRUE(may_exist); ASSERT_TRUE(iter->Valid()); - ASSERT_EQ(options.comparator->Compare( - iter->key(), ikey.Encode().ToString()), 0); + ASSERT_EQ( + options.comparator->Compare(iter->key(), ikey.Encode().ToString()), 0); ASSERT_EQ(iter->value(), value); } @@ -419,8 +417,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) { ASSERT_TRUE(iter->Valid()); // user key should match - ASSERT_EQ(options.comparator->Compare( - ExtractUserKey(iter->key()), ukey), 0); + ASSERT_EQ(options.comparator->Compare(ExtractUserKey(iter->key()), ukey), + 0); // seek_key seqno number should be greater than that of iter result ASSERT_GT(GetInternalKeySeqno(seek_ikey.Encode()), @@ -601,7 +599,7 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2, ReadOptions ro; ASSERT_OK(table_reader->Get(ro, seek_ikey.Encode().ToString(), &get_context, - moptions.prefix_extractor.get())); + moptions.prefix_extractor.get())); } TEST(DataBlockHashIndex, BlockBoundary) { diff --git a/table/table_test.cc b/table/table_test.cc index 3a5408622..af648bc05 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -3660,7 +3660,7 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) { BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); table_options.data_block_index_type = - BlockBasedTableOptions::kDataBlockBinaryAndHash; + BlockBasedTableOptions::kDataBlockBinaryAndHash; Options options; options.comparator = BytewiseComparator(); @@ -3685,14 +3685,12 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) { c.Finish(options, ioptions, moptions, table_options, internal_comparator, &keys, &kvmap); - auto reader = c.GetTableReader(); std::unique_ptr seek_iter; - seek_iter.reset(reader->NewIterator(ReadOptions(), - moptions.prefix_extractor.get())); + seek_iter.reset( + reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get())); for (int i = 0; i < 2; ++i) { - ReadOptions ro; // for every kv, we seek using two method: Get() and Seek() // Get() will use the SuffixIndexHash in Block. For non-existent key it @@ -3727,17 +3725,17 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) { // Search for non-existent keys for (auto& kv : kvmap) { std::string user_key = ExtractUserKey(kv.first).ToString(); - user_key.back() = '0'; // make it non-existent key + user_key.back() = '0'; // make it non-existent key InternalKey internal_key(user_key, 0, kTypeValue); std::string encoded_key = internal_key.Encode().ToString(); - if (i == 0) { // Search using Seek() + if (i == 0) { // Search using Seek() seek_iter->Seek(encoded_key); ASSERT_OK(seek_iter->status()); - if (seek_iter->Valid()){ + if (seek_iter->Valid()) { ASSERT_TRUE(BytewiseComparator()->Compare( user_key, ExtractUserKey(seek_iter->key())) < 0); } - } else { // Search using Get() + } else { // Search using Get() PinnableSlice value; GetContext get_context(options.comparator, nullptr, nullptr, nullptr, GetContext::kNotFound, user_key, &value, nullptr, diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 79a84766a..c32822c8f 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -466,6 +466,16 @@ DEFINE_bool(enable_index_compression, DEFINE_bool(block_align, rocksdb::BlockBasedTableOptions().block_align, "Align data blocks on page size"); +DEFINE_bool(use_data_block_hash_index, false, + "if use kDataBlockBinaryAndHash " + "instead of kDataBlockBinarySearch. " + "This is valid if only we use BlockTable"); + +DEFINE_double(data_block_hash_table_util_ratio, 0.75, + "util ratio for data block hash index table. " + "This is only valid if use_data_block_hash_index is " + "set to true"); + DEFINE_int64(compressed_cache_size, -1, "Number of bytes to use as a cache of compressed data."); @@ -3265,6 +3275,15 @@ void VerifyDBFromDB(std::string& truth_db_name) { block_based_options.enable_index_compression = FLAGS_enable_index_compression; block_based_options.block_align = FLAGS_block_align; + if (FLAGS_use_data_block_hash_index) { + block_based_options.data_block_index_type = + rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash; + } else { + block_based_options.data_block_index_type = + rocksdb::BlockBasedTableOptions::kDataBlockBinarySearch; + } + block_based_options.data_block_hash_table_util_ratio = + FLAGS_data_block_hash_table_util_ratio; if (FLAGS_read_cache_path != "") { #ifndef ROCKSDB_LITE Status rc_status;