Add db_bench options of data block hash index (#4281)

Summary:
Add `--data_block_index_type` and `--data_block_hash_table_util_ratio` option to `db_bench`.

`--data_block_index_type` can be either of `binary` (default) or `binary_and_hash`;
`--data_block_hash_table_util_ratio` will be a double. The default value is `0.75`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4281

Differential Revision: D9361476

Pulled By: fgwu

fbshipit-source-id: dc53e01acef9db81b9eec5e8a96f3bc8ed718c10

include/rocksdb/table.h

@@ -112,7 +112,6 @@ struct BlockBasedTableOptions {
   // kDataBlockBinaryAndHash.
   double data_block_hash_table_util_ratio = 0.75;
 
-
   // This option is now deprecated. No matter what value it is set to,
   // it will behave as if hash_index_allow_collision=true.
   bool hash_index_allow_collision = true;

table/block.cc

@@ -789,8 +789,8 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno,
           uint16_t map_offset;
           data_block_hash_index_.Initialize(
               contents.data.data(),
-              static_cast<uint16_t>(
+              static_cast<uint16_t>(contents.data.size() -
-                  contents.data.size() - sizeof(uint32_t)),     /*chop off
+                                    sizeof(uint32_t)), /*chop off
                                                    NUM_RESTARTS*/
               &map_offset);
 

table/block_builder.cc

@@ -33,8 +33,8 @@
 
 #include "table/block_builder.h"
 
-#include <algorithm>
 #include <assert.h>
+#include <algorithm>
 #include "db/dbformat.h"
 #include "rocksdb/comparator.h"
 #include "table/data_block_footer.h"
@@ -126,8 +126,7 @@ Slice BlockBuilder::Finish() {
   }
 
   // footer is a packed format of data_block_index_type and num_restarts
-  uint32_t block_footer = PackIndexTypeAndNumRestarts(
+  uint32_t block_footer = PackIndexTypeAndNumRestarts(index_type, num_restarts);
-      index_type, num_restarts);
 
   PutFixed32(&buffer_, block_footer);
   finished_ = true;

table/data_block_footer.cc

@@ -38,7 +38,6 @@ uint32_t PackIndexTypeAndNumRestarts(
   return block_footer;
 }
 
-
 void UnPackIndexTypeAndNumRestarts(
     uint32_t block_footer,
     BlockBasedTableOptions::DataBlockIndexType* index_type,

table/data_block_footer.h

@@ -17,7 +17,6 @@ uint32_t PackIndexTypeAndNumRestarts(
     BlockBasedTableOptions::DataBlockIndexType index_type,
     uint32_t num_restarts);
 
-
 void UnPackIndexTypeAndNumRestarts(
     uint32_t block_footer,
     BlockBasedTableOptions::DataBlockIndexType* index_type,

table/data_block_hash_index.cc

@@ -54,8 +54,8 @@ void DataBlockHashIndexBuilder::Finish(std::string& buffer) {
   }
 
   for (uint8_t restart_index : buckets) {
-    buffer.append(const_cast<const char*>(
+    buffer.append(
-                      reinterpret_cast<char*>(&restart_index)),
+        const_cast<const char*>(reinterpret_cast<char*>(&restart_index)),
         sizeof(restart_index));
   }
 

table/data_block_hash_index_test.cc

@@ -9,10 +9,10 @@
 
 #include "rocksdb/slice.h"
 #include "table/block.h"
+#include "table/block_based_table_reader.h"
 #include "table/block_builder.h"
 #include "table/data_block_hash_index.h"
 #include "table/get_context.h"
-#include "table/block_based_table_reader.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
 
@@ -40,9 +40,9 @@ static std::string RandomString(Random* rnd, int len) {
   return r;
 }
 std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
-                        Random *rnd) {
+                        Random* rnd) {
   char buf[50];
-  char *p = &buf[0];
+  char* p = &buf[0];
   snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key);
   std::string k(p);
   if (padding_size) {
@@ -55,8 +55,8 @@ std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
 // Generate random key value pairs.
 // The generated key will be sorted. You can tune the parameters to generated
 // different kinds of test key/value pairs for different scenario.
-void GenerateRandomKVs(std::vector<std::string> *keys,
+void GenerateRandomKVs(std::vector<std::string>* keys,
-                       std::vector<std::string> *values, const int from,
+                       std::vector<std::string>* values, const int from,
                        const int len, const int step = 1,
                        const int padding_size = 0,
                        const int keys_share_prefix = 1) {
@@ -95,7 +95,6 @@ TEST(DataBlockHashIndex, DataBlockHashTestSmall) {
 
     buffer2 = buffer;  // test for the correctness of relative offset
 
-
     Slice s(buffer2);
     DataBlockHashIndex index;
     uint16_t map_offset;
@@ -290,7 +289,6 @@ TEST(DataBlockHashIndex, BlockRestartIndexExceedMax) {
               BlockBasedTableOptions::kDataBlockBinaryAndHash);
   }
 
-
   builder.Reset();
 
   // #restarts > 253. HashIndex is not used
@@ -404,8 +402,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
     may_exist = iter->SeekForGet(seek_ikey.Encode().ToString());
     ASSERT_TRUE(may_exist);
     ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ(options.comparator->Compare(
+    ASSERT_EQ(
-                  iter->key(), ikey.Encode().ToString()), 0);
+        options.comparator->Compare(iter->key(), ikey.Encode().ToString()), 0);
     ASSERT_EQ(iter->value(), value);
   }
 
@@ -419,8 +417,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
     ASSERT_TRUE(iter->Valid());
 
     // user key should match
-    ASSERT_EQ(options.comparator->Compare(
+    ASSERT_EQ(options.comparator->Compare(ExtractUserKey(iter->key()), ukey),
-                  ExtractUserKey(iter->key()), ukey), 0);
+              0);
 
     // seek_key seqno number should be greater than that of iter result
     ASSERT_GT(GetInternalKeySeqno(seek_ikey.Encode()),

table/table_test.cc

@@ -3685,14 +3685,12 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
   c.Finish(options, ioptions, moptions, table_options, internal_comparator,
            &keys, &kvmap);
 
-
   auto reader = c.GetTableReader();
 
   std::unique_ptr<InternalIterator> seek_iter;
-  seek_iter.reset(reader->NewIterator(ReadOptions(),
+  seek_iter.reset(
-                                      moptions.prefix_extractor.get()));
+      reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get()));
   for (int i = 0; i < 2; ++i) {
-
     ReadOptions ro;
     // for every kv, we seek using two method: Get() and Seek()
     // Get() will use the SuffixIndexHash in Block. For non-existent key it
@@ -3733,7 +3731,7 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
       if (i == 0) {  // Search using Seek()
         seek_iter->Seek(encoded_key);
         ASSERT_OK(seek_iter->status());
-        if (seek_iter->Valid()){
+        if (seek_iter->Valid()) {
           ASSERT_TRUE(BytewiseComparator()->Compare(
                           user_key, ExtractUserKey(seek_iter->key())) < 0);
         }

tools/db_bench_tool.cc

@@ -466,6 +466,16 @@ DEFINE_bool(enable_index_compression,
 DEFINE_bool(block_align, rocksdb::BlockBasedTableOptions().block_align,
             "Align data blocks on page size");
 
+DEFINE_bool(use_data_block_hash_index, false,
+            "if use kDataBlockBinaryAndHash "
+            "instead of kDataBlockBinarySearch. "
+            "This is valid if only we use BlockTable");
+
+DEFINE_double(data_block_hash_table_util_ratio, 0.75,
+              "util ratio for data block hash index table. "
+              "This is only valid if use_data_block_hash_index is "
+              "set to true");
+
 DEFINE_int64(compressed_cache_size, -1,
              "Number of bytes to use as a cache of compressed data.");
 
@@ -3265,6 +3275,15 @@ void VerifyDBFromDB(std::string& truth_db_name) {
       block_based_options.enable_index_compression =
           FLAGS_enable_index_compression;
       block_based_options.block_align = FLAGS_block_align;
+      if (FLAGS_use_data_block_hash_index) {
+        block_based_options.data_block_index_type =
+            rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash;
+      } else {
+        block_based_options.data_block_index_type =
+            rocksdb::BlockBasedTableOptions::kDataBlockBinarySearch;
+      }
+      block_based_options.data_block_hash_table_util_ratio =
+          FLAGS_data_block_hash_table_util_ratio;
       if (FLAGS_read_cache_path != "") {
 #ifndef ROCKSDB_LITE
         Status rc_status;