Move the uncompression dictionary object out of the block cache (#5584)
Summary: RocksDB has historically stored uncompression dictionary objects in the block cache as opposed to storing just the block contents. This neccesitated evicting the object upon table close. With the new code, only the raw blocks are stored in the cache, eliminating the need for eviction. In addition, the patch makes the following improvements: 1) Compression dictionary blocks are now prefetched/pinned similarly to index/filter blocks. 2) A copy operation got eliminated when the uncompression dictionary is retrieved. 3) Errors related to retrieving the uncompression dictionary are propagated as opposed to silently ignored. Note: the patch temporarily breaks the compression dictionary evicition stats. They will be fixed in a separate phase. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5584 Test Plan: make asan_check Differential Revision: D16344151 Pulled By: ltamasi fbshipit-source-id: 2962b295f5b19628f9da88a3fcebbce5a5017a7bmain
Levi Tamasi
5 years ago
committed by
Facebook Github Bot
parent
6b7fcc0d5f
commit
092f417037
@ -0,0 +1,138 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
|
||||
#include "table/block_based/uncompression_dict_reader.h" |
||||
#include "monitoring/perf_context_imp.h" |
||||
#include "table/block_based/block_based_table_reader.h" |
||||
#include "util/compression.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
Status UncompressionDictReader::Create( |
||||
const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, |
||||
bool use_cache, bool prefetch, bool pin, |
||||
BlockCacheLookupContext* lookup_context, |
||||
std::unique_ptr<UncompressionDictReader>* uncompression_dict_reader) { |
||||
assert(table); |
||||
assert(table->get_rep()); |
||||
assert(!pin || prefetch); |
||||
assert(uncompression_dict_reader); |
||||
|
||||
CachableEntry<BlockContents> uncompression_dict_block; |
||||
if (prefetch || !use_cache) { |
||||
const Status s = ReadUncompressionDictionaryBlock( |
||||
table, prefetch_buffer, ReadOptions(), nullptr /* get_context */, |
||||
lookup_context, &uncompression_dict_block); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
if (use_cache && !pin) { |
||||
uncompression_dict_block.Reset(); |
||||
} |
||||
} |
||||
|
||||
uncompression_dict_reader->reset( |
||||
new UncompressionDictReader(table, std::move(uncompression_dict_block))); |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status UncompressionDictReader::ReadUncompressionDictionaryBlock( |
||||
const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, |
||||
const ReadOptions& read_options, GetContext* get_context, |
||||
BlockCacheLookupContext* lookup_context, |
||||
CachableEntry<BlockContents>* uncompression_dict_block) { |
||||
// TODO: add perf counter for compression dictionary read time
|
||||
|
||||
assert(table); |
||||
assert(uncompression_dict_block); |
||||
assert(uncompression_dict_block->IsEmpty()); |
||||
|
||||
const BlockBasedTable::Rep* const rep = table->get_rep(); |
||||
assert(rep); |
||||
assert(!rep->compression_dict_handle.IsNull()); |
||||
|
||||
const Status s = table->RetrieveBlock( |
||||
prefetch_buffer, read_options, rep->compression_dict_handle, |
||||
UncompressionDict::GetEmptyDict(), uncompression_dict_block, |
||||
BlockType::kCompressionDictionary, get_context, lookup_context); |
||||
|
||||
if (!s.ok()) { |
||||
ROCKS_LOG_WARN( |
||||
rep->ioptions.info_log, |
||||
"Encountered error while reading data from compression dictionary " |
||||
"block %s", |
||||
s.ToString().c_str()); |
||||
} |
||||
|
||||
return s; |
||||
} |
||||
|
||||
Status UncompressionDictReader::GetOrReadUncompressionDictionaryBlock( |
||||
FilePrefetchBuffer* prefetch_buffer, bool no_io, GetContext* get_context, |
||||
BlockCacheLookupContext* lookup_context, |
||||
CachableEntry<BlockContents>* uncompression_dict_block) const { |
||||
assert(uncompression_dict_block); |
||||
|
||||
if (!uncompression_dict_block_.IsEmpty()) { |
||||
uncompression_dict_block->SetUnownedValue( |
||||
uncompression_dict_block_.GetValue()); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
ReadOptions read_options; |
||||
if (no_io) { |
||||
read_options.read_tier = kBlockCacheTier; |
||||
} |
||||
|
||||
return ReadUncompressionDictionaryBlock(table_, prefetch_buffer, read_options, |
||||
get_context, lookup_context, |
||||
uncompression_dict_block); |
||||
} |
||||
|
||||
Status UncompressionDictReader::GetOrReadUncompressionDictionary( |
||||
FilePrefetchBuffer* prefetch_buffer, bool no_io, GetContext* get_context, |
||||
BlockCacheLookupContext* lookup_context, |
||||
UncompressionDict* uncompression_dict) const { |
||||
CachableEntry<BlockContents> uncompression_dict_block; |
||||
const Status s = GetOrReadUncompressionDictionaryBlock( |
||||
prefetch_buffer, no_io, get_context, lookup_context, |
||||
&uncompression_dict_block); |
||||
|
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
assert(uncompression_dict); |
||||
assert(table_); |
||||
assert(table_->get_rep()); |
||||
|
||||
UncompressionDict dict(uncompression_dict_block.GetValue()->data, |
||||
table_->get_rep()->blocks_definitely_zstd_compressed); |
||||
*uncompression_dict = std::move(dict); |
||||
uncompression_dict_block.TransferTo(uncompression_dict); |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
size_t UncompressionDictReader::ApproximateMemoryUsage() const { |
||||
assert(!uncompression_dict_block_.GetOwnValue() || |
||||
uncompression_dict_block_.GetValue() != nullptr); |
||||
size_t usage = uncompression_dict_block_.GetOwnValue() |
||||
? uncompression_dict_block_.GetValue()->ApproximateMemoryUsage() |
||||
: 0; |
||||
|
||||
#ifdef ROCKSDB_MALLOC_USABLE_SIZE |
||||
usage += malloc_usable_size(const_cast<UncompressionDictReader*>(this)); |
||||
#else |
||||
usage += sizeof(*this); |
||||
#endif // ROCKSDB_MALLOC_USABLE_SIZE
|
||||
|
||||
return usage; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,64 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
|
||||
#pragma once |
||||
|
||||
#include <cassert> |
||||
#include "table/block_based/cachable_entry.h" |
||||
#include "table/format.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class BlockBasedTable; |
||||
struct BlockCacheLookupContext; |
||||
class FilePrefetchBuffer; |
||||
class GetContext; |
||||
struct ReadOptions; |
||||
struct UncompressionDict; |
||||
|
||||
// Provides access to the uncompression dictionary regardless of whether
|
||||
// it is owned by the reader or stored in the cache, or whether it is pinned
|
||||
// in the cache or not.
|
||||
class UncompressionDictReader { |
||||
public: |
||||
static Status Create( |
||||
const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, |
||||
bool use_cache, bool prefetch, bool pin, |
||||
BlockCacheLookupContext* lookup_context, |
||||
std::unique_ptr<UncompressionDictReader>* uncompression_dict_reader); |
||||
|
||||
Status GetOrReadUncompressionDictionary( |
||||
FilePrefetchBuffer* prefetch_buffer, bool no_io, GetContext* get_context, |
||||
BlockCacheLookupContext* lookup_context, |
||||
UncompressionDict* uncompression_dict) const; |
||||
|
||||
size_t ApproximateMemoryUsage() const; |
||||
|
||||
private: |
||||
UncompressionDictReader( |
||||
const BlockBasedTable* t, |
||||
CachableEntry<BlockContents>&& uncompression_dict_block) |
||||
: table_(t), |
||||
uncompression_dict_block_(std::move(uncompression_dict_block)) { |
||||
assert(table_); |
||||
} |
||||
|
||||
static Status ReadUncompressionDictionaryBlock( |
||||
const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, |
||||
const ReadOptions& read_options, GetContext* get_context, |
||||
BlockCacheLookupContext* lookup_context, |
||||
CachableEntry<BlockContents>* uncompression_dict_block); |
||||
|
||||
Status GetOrReadUncompressionDictionaryBlock( |
||||
FilePrefetchBuffer* prefetch_buffer, bool no_io, GetContext* get_context, |
||||
BlockCacheLookupContext* lookup_context, |
||||
CachableEntry<BlockContents>* uncompression_dict_block) const; |
||||
|
||||
const BlockBasedTable* table_; |
||||
CachableEntry<BlockContents> uncompression_dict_block_; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
Loading…
Reference in new issue