NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Evict the uncompression dictionary from the block cache upon table close (#5150)

Browse Source 
Summary:
The uncompression dictionary object has a Statistics pointer that might
dangle if the database closed. This patch evicts the dictionary from the
block cache when a table is closed, similarly to how index and filter
readers are handled.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5150

Differential Revision: D14782422

Pulled By: ltamasi

fbshipit-source-id: 0cec9336c742c479aa92206e04521767f1aa9622
main
Levi Tamasi 6 years ago committed by Facebook Github Bot
parent 306b9adfd8
commit 59ef2ba559
2 changed files with 130 additions and 86 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 25
      table/block_based_table_reader.cc
  2. 67
      table/table_test.cc

25
table/block_based_table_reader.cc
Unescape View File

@ -3275,21 +3275,36 @@ void BlockBasedTable::Close() {
if (rep_->closed) { if (rep_->closed) {
return; return;
} }
rep_->filter_entry.Release(rep_->table_options.block_cache.get());
rep_->index_entry.Release(rep_->table_options.block_cache.get()); Cache* const cache = rep_->table_options.block_cache.get();
// cleanup index and filter blocks to avoid accessing dangling pointer
rep_->filter_entry.Release(cache);
rep_->index_entry.Release(cache);
// cleanup index, filter, and compression dictionary blocks
// to avoid accessing dangling pointers
if (!rep_->table_options.no_block_cache) { if (!rep_->table_options.no_block_cache) {
char cache_key[kMaxCacheKeyPrefixSize + kMaxVarint64Length]; char cache_key[kMaxCacheKeyPrefixSize + kMaxVarint64Length];
// Get the filter block key // Get the filter block key
auto key = GetCacheKey(rep_->cache_key_prefix, rep_->cache_key_prefix_size, auto key = GetCacheKey(rep_->cache_key_prefix, rep_->cache_key_prefix_size,
rep_->filter_handle, cache_key); rep_->filter_handle, cache_key);
rep_->table_options.block_cache.get()->Erase(key); cache->Erase(key);
// Get the index block key // Get the index block key
key = GetCacheKeyFromOffset(rep_->cache_key_prefix, key = GetCacheKeyFromOffset(rep_->cache_key_prefix,
rep_->cache_key_prefix_size, rep_->cache_key_prefix_size,
rep_->dummy_index_reader_offset, cache_key); rep_->dummy_index_reader_offset, cache_key);
rep_->table_options.block_cache.get()->Erase(key); cache->Erase(key);
if (!rep_->compression_dict_handle.IsNull()) {
// Get the compression dictionary block key
key = GetCacheKey(rep_->cache_key_prefix, rep_->cache_key_prefix_size,
rep_->compression_dict_handle, cache_key);
cache->Erase(key);
} }
}
rep_->closed = true; rep_->closed = true;
} }

67
table/table_test.cc
Unescape View File

@ -329,7 +329,7 @@ class TableConstructor: public Constructor {
TableBuilderOptions(ioptions, moptions, internal_comparator, TableBuilderOptions(ioptions, moptions, internal_comparator,
&int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression, options.compression, options.sample_for_compression,
CompressionOptions(), false /* skip_filters */, options.compression_opts, false /* skip_filters */,
column_family_name, level_), column_family_name, level_),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer_.get())); file_writer_.get()));
@ -2317,6 +2317,22 @@ std::map<std::string, size_t> MockCache::marked_data_in_cache_;
// table is closed. This test makes sure that the only items remains in the // table is closed. This test makes sure that the only items remains in the
// cache after the table is closed are raw data blocks. // cache after the table is closed are raw data blocks.
TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) { TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
std::vector<CompressionType> compression_types{kNoCompression};
// The following are the compression library versions supporting compression
// dictionaries. See the test case CacheCompressionDict in the
// DBBlockCacheTest suite.
#ifdef ZLIB
compression_types.push_back(kZlibCompression);
#endif // ZLIB
#if LZ4_VERSION_NUMBER >= 10400
compression_types.push_back(kLZ4Compression);
compression_types.push_back(kLZ4HCCompression);
#endif // LZ4_VERSION_NUMBER >= 10400
#if ZSTD_VERSION_NUMBER >= 500
compression_types.push_back(kZSTD);
#endif // ZSTD_VERSION_NUMBER >= 500
for (int level: {-1, 0, 1, 10}) { for (int level: {-1, 0, 1, 10}) {
for (auto index_type : for (auto index_type :
{BlockBasedTableOptions::IndexType::kBinarySearch, {BlockBasedTableOptions::IndexType::kBinarySearch,
@ -2335,31 +2351,41 @@ TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
if (pin_l0 && !index_and_filter_in_cache) { if (pin_l0 && !index_and_filter_in_cache) {
continue; continue;
} }
for (auto compression_type : compression_types) {
for (uint32_t max_dict_bytes : {0, 1 << 14}) {
if (compression_type == kNoCompression && max_dict_bytes)
continue;
// Create a table // Create a table
Options opt; Options opt;
std::unique_ptr<InternalKeyComparator> ikc; std::unique_ptr<InternalKeyComparator> ikc;
ikc.reset(new test::PlainInternalKeyComparator(opt.comparator)); ikc.reset(new test::PlainInternalKeyComparator(
opt.compression = kNoCompression; opt.comparator));
opt.compression = compression_type;
opt.compression_opts.max_dict_bytes = max_dict_bytes;
BlockBasedTableOptions table_options = BlockBasedTableOptions table_options =
GetBlockBasedTableOptions(); GetBlockBasedTableOptions();
table_options.block_size = 1024; table_options.block_size = 1024;
table_options.index_type = table_options.index_type = index_type;
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; table_options.pin_l0_filter_and_index_blocks_in_cache =
table_options.pin_l0_filter_and_index_blocks_in_cache = pin_l0; pin_l0;
table_options.pin_top_level_index_and_filter = pin_top_level; table_options.pin_top_level_index_and_filter =
pin_top_level;
table_options.partition_filters = partition_filter; table_options.partition_filters = partition_filter;
table_options.cache_index_and_filter_blocks = table_options.cache_index_and_filter_blocks =
index_and_filter_in_cache; index_and_filter_in_cache;
// big enough so we don't ever lose cached values. // big enough so we don't ever lose cached values.
table_options.block_cache = std::shared_ptr<rocksdb::Cache>( table_options.block_cache = std::make_shared<MockCache>(
new MockCache(16 * 1024 * 1024, 4, false, 0.0)); 16 * 1024 * 1024, 4, false, 0.0);
table_options.filter_policy.reset( table_options.filter_policy.reset(
rocksdb::NewBloomFilterPolicy(10, block_based_filter)); rocksdb::NewBloomFilterPolicy(10, block_based_filter));
opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); opt.table_factory.reset(NewBlockBasedTableFactory(
table_options));
bool convert_to_internal_key = false; bool convert_to_internal_key = false;
TableConstructor c(BytewiseComparator(), convert_to_internal_key, TableConstructor c(BytewiseComparator(),
level); convert_to_internal_key, level);
std::string user_key = "k01"; std::string user_key = "k01";
std::string key = std::string key =
InternalKey(user_key, 0, kTypeValue).Encode().ToString(); InternalKey(user_key, 0, kTypeValue).Encode().ToString();
@ -2368,15 +2394,15 @@ TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
stl_wrappers::KVMap kvmap; stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(opt); const ImmutableCFOptions ioptions(opt);
const MutableCFOptions moptions(opt); const MutableCFOptions moptions(opt);
c.Finish(opt, ioptions, moptions, table_options, *ikc, &keys, c.Finish(opt, ioptions, moptions, table_options, *ikc,
&kvmap); &keys, &kvmap);
// Doing a read to make index/filter loaded into the cache // Doing a read to make index/filter loaded into the cache
auto table_reader = auto table_reader =
dynamic_cast<BlockBasedTable*>(c.GetTableReader()); dynamic_cast<BlockBasedTable*>(c.GetTableReader());
PinnableSlice value; PinnableSlice value;
GetContext get_context(opt.comparator, nullptr, nullptr, nullptr, GetContext get_context(opt.comparator, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr, GetContext::kNotFound, user_key, &value,
nullptr, nullptr, nullptr, nullptr); nullptr, nullptr, nullptr, nullptr);
InternalKey ikey(user_key, 0, kTypeValue); InternalKey ikey(user_key, 0, kTypeValue);
auto s = table_reader->Get(ReadOptions(), key, &get_context, auto s = table_reader->Get(ReadOptions(), key, &get_context,
@ -2388,11 +2414,12 @@ TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
c.ResetTableReader(); c.ResetTableReader();
auto usage = table_options.block_cache->GetUsage(); auto usage = table_options.block_cache->GetUsage();
auto pinned_usage = table_options.block_cache->GetPinnedUsage(); auto pinned_usage =
table_options.block_cache->GetPinnedUsage();
// The only usage must be for marked data blocks // The only usage must be for marked data blocks
ASSERT_EQ(usage, MockCache::marked_size_); ASSERT_EQ(usage, MockCache::marked_size_);
// There must be some pinned data since PinnableSlice has not // There must be some pinned data since PinnableSlice has
// released them yet // not released them yet
ASSERT_GT(pinned_usage, 0); ASSERT_GT(pinned_usage, 0);
// Release pinnable slice reousrces // Release pinnable slice reousrces
value.Reset(); value.Reset();
@ -2404,6 +2431,8 @@ TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
} }
} }
} }
}
}
} // level } // level
} }

Write Preview
Loading…
Cancel
Save