Add a BlockBasedTableOption to turn off index block compression.

Summary:
Add a new bool option index_uncompressed in BlockBasedTableOptions.
Closes https://github.com/facebook/rocksdb/pull/3303

Differential Revision: D6686161

Pulled By: anand1976

fbshipit-source-id: 748b46993d48a01e5f89b6bd3e41f06a59ec6054
main
Anand Ananthabhotla 7 years ago committed by Facebook Github Bot
parent bafec6bb30
commit 199405192d
  1. 5
      include/rocksdb/table.h
  2. 3
      options/options_settable_test.cc
  3. 16
      table/block_based_table_builder.cc
  4. 3
      table/block_based_table_factory.cc
  5. 5
      table/block_based_table_factory.h
  6. 44
      table/table_test.cc
  7. 6
      tools/db_bench_tool.cc

@ -217,6 +217,11 @@ struct BlockBasedTableOptions {
// This option only affects newly written tables. When reading exising tables, // This option only affects newly written tables. When reading exising tables,
// the information about version is read from the footer. // the information about version is read from the footer.
uint32_t format_version = 2; uint32_t format_version = 2;
// Store index blocks on disk in compressed format. Changing this option to
// false will avoid the overhead of decompression if index blocks are evicted
// and read back
bool enable_index_compression = true;
}; };
// Table Properties that are specific to block-based table properties. // Table Properties that are specific to block-based table properties.

@ -150,7 +150,8 @@ TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
"filter_policy=bloomfilter:4:true;whole_key_filtering=1;" "filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
"format_version=1;" "format_version=1;"
"hash_index_allow_collision=false;" "hash_index_allow_collision=false;"
"verify_compression=true;read_amp_bytes_per_bit=0", "verify_compression=true;read_amp_bytes_per_bit=0;"
"enable_index_compression=false",
new_bbto)); new_bbto));
ASSERT_EQ(unset_bytes_base, ASSERT_EQ(unset_bytes_base,

@ -783,9 +783,12 @@ Status BlockBasedTableBuilder::Finish() {
WriteRawBlock(meta_index_builder.Finish(), kNoCompression, WriteRawBlock(meta_index_builder.Finish(), kNoCompression,
&metaindex_block_handle); &metaindex_block_handle);
const bool is_data_block = true; if (r->table_options.enable_index_compression) {
WriteBlock(index_blocks.index_block_contents, &index_block_handle, WriteBlock(index_blocks.index_block_contents, &index_block_handle, false);
!is_data_block); } else {
WriteRawBlock(index_blocks.index_block_contents, kNoCompression,
&index_block_handle);
}
// If there are more index partitions, finish them and write them out // If there are more index partitions, finish them and write them out
Status& s = index_builder_status; Status& s = index_builder_status;
while (s.IsIncomplete()) { while (s.IsIncomplete()) {
@ -793,8 +796,13 @@ Status BlockBasedTableBuilder::Finish() {
if (!s.ok() && !s.IsIncomplete()) { if (!s.ok() && !s.IsIncomplete()) {
return s; return s;
} }
if (r->table_options.enable_index_compression) {
WriteBlock(index_blocks.index_block_contents, &index_block_handle, WriteBlock(index_blocks.index_block_contents, &index_block_handle,
!is_data_block); false);
} else {
WriteRawBlock(index_blocks.index_block_contents, kNoCompression,
&index_block_handle);
}
// The last index_block_handle will be for the partition index block // The last index_block_handle will be for the partition index block
} }
} }

@ -223,6 +223,9 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const {
snprintf(buffer, kBufferSize, " format_version: %d\n", snprintf(buffer, kBufferSize, " format_version: %d\n",
table_options_.format_version); table_options_.format_version);
ret.append(buffer); ret.append(buffer);
snprintf(buffer, kBufferSize, " enable_index_compression: %d\n",
table_options_.enable_index_compression);
ret.append(buffer);
return ret; return ret;
} }

@ -152,6 +152,9 @@ static std::unordered_map<std::string, OptionTypeInfo>
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"read_amp_bytes_per_bit", {"read_amp_bytes_per_bit",
{offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}}; OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"enable_index_compression",
{offsetof(struct BlockBasedTableOptions, enable_index_compression),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}};
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
} // namespace rocksdb } // namespace rocksdb

@ -1003,7 +1003,10 @@ class TableTest : public testing::Test {
}; };
class GeneralTableTest : public TableTest {}; class GeneralTableTest : public TableTest {};
class BlockBasedTableTest : public TableTest {}; class BlockBasedTableTest : public TableTest {
protected:
uint64_t IndexUncompressedHelper(bool indexCompress);
};
class PlainTableTest : public TableTest {}; class PlainTableTest : public TableTest {};
class TablePropertyTest : public testing::Test {}; class TablePropertyTest : public testing::Test {};
@ -1064,13 +1067,17 @@ TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) {
stl_wrappers::KVMap kvmap; stl_wrappers::KVMap kvmap;
Options options; Options options;
options.compression = kNoCompression; options.compression = kNoCompression;
options.statistics = CreateDBStatistics();
options.statistics->stats_level_ = StatsLevel::kAll;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.block_restart_interval = 1; table_options.block_restart_interval = 1;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options); ImmutableCFOptions ioptions(options);
ioptions.statistics = options.statistics.get();
c.Finish(options, ioptions, table_options, c.Finish(options, ioptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap); GetPlainInternalComparator(options.comparator), &keys, &kvmap);
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_NOT_COMPRESSED), 0);
auto& props = *c.GetTableReader()->GetTableProperties(); auto& props = *c.GetTableReader()->GetTableProperties();
ASSERT_EQ(kvmap.size(), props.num_entries); ASSERT_EQ(kvmap.size(), props.num_entries);
@ -1094,6 +1101,39 @@ TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) {
c.ResetTableReader(); c.ResetTableReader();
} }
uint64_t BlockBasedTableTest::IndexUncompressedHelper(bool compressed) {
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
constexpr size_t kNumKeys = 10000;
for (size_t k = 0; k < kNumKeys; ++k) {
c.Add("key" + ToString(k), "val" + ToString(k));
}
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
Options options;
options.compression = kSnappyCompression;
options.statistics = CreateDBStatistics();
options.statistics->stats_level_ = StatsLevel::kAll;
BlockBasedTableOptions table_options;
table_options.block_restart_interval = 1;
table_options.enable_index_compression = compressed;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
ImmutableCFOptions ioptions(options);
ioptions.statistics = options.statistics.get();
c.Finish(options, ioptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
c.ResetTableReader();
return options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
}
TEST_F(BlockBasedTableTest, IndexUncompressed) {
uint64_t tbl1_compressed_cnt = IndexUncompressedHelper(true);
uint64_t tbl2_compressed_cnt = IndexUncompressedHelper(false);
// tbl1_compressed_cnt should include 1 index block
EXPECT_EQ(tbl2_compressed_cnt + 1, tbl1_compressed_cnt);
}
TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) { TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) {
TableConstructor c(&reverse_key_comparator); TableConstructor c(&reverse_key_comparator);
std::vector<std::string> keys; std::vector<std::string> keys;

@ -442,6 +442,10 @@ DEFINE_int32(read_amp_bytes_per_bit,
rocksdb::BlockBasedTableOptions().read_amp_bytes_per_bit, rocksdb::BlockBasedTableOptions().read_amp_bytes_per_bit,
"Number of bytes per bit to be used in block read-amp bitmap"); "Number of bytes per bit to be used in block read-amp bitmap");
DEFINE_bool(enable_index_compression,
rocksdb::BlockBasedTableOptions().enable_index_compression,
"Compress the index block");
DEFINE_int64(compressed_cache_size, -1, DEFINE_int64(compressed_cache_size, -1,
"Number of bytes to use as a cache of compressed data."); "Number of bytes to use as a cache of compressed data.");
@ -3110,6 +3114,8 @@ void VerifyDBFromDB(std::string& truth_db_name) {
block_based_options.filter_policy = filter_policy_; block_based_options.filter_policy = filter_policy_;
block_based_options.format_version = 2; block_based_options.format_version = 2;
block_based_options.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit; block_based_options.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit;
block_based_options.enable_index_compression =
FLAGS_enable_index_compression;
if (FLAGS_read_cache_path != "") { if (FLAGS_read_cache_path != "") {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
Status rc_status; Status rc_status;

Loading…
Cancel
Save