Only cache level 0 indexes and filter when opening table reader

Summary: In T8216281 we decided to disable prefetching the index and filter during opening table handlers during startup (max_open_files = -1).

Test Plan: Rely on `IndexAndFilterBlocksOfNewTableAddedToCache` to guarantee L0 indexes and filters are still cached and change `PinL0IndexAndFilterBlocksTest` to make sure other levels are not cached (maybe add one more test to test we don't cache other levels?)

Reviewers: sdong, andrewkr

Reviewed By: andrewkr

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D59913
main
omegaga 8 years ago
parent 7bedd94406
commit e70020e4f6
  1. 126
      db/db_test2.cc
  2. 9
      db/plain_table_db_test.cc
  3. 10
      db/table_cache.cc
  4. 6
      db/table_cache.h
  5. 22
      db/version_builder.cc
  6. 3
      db/version_builder.h
  7. 8
      db/version_set.cc
  8. 3
      include/rocksdb/table.h
  9. 3
      table/adaptive_table_factory.cc
  10. 9
      table/adaptive_table_factory.h
  11. 14
      table/block_based_table_factory.cc
  12. 17
      table/block_based_table_factory.h
  13. 39
      table/block_based_table_reader.cc
  14. 10
      table/block_based_table_reader.h
  15. 3
      table/cuckoo_table_factory.cc
  16. 9
      table/cuckoo_table_factory.h
  17. 3
      table/mock_table.cc
  18. 9
      table/mock_table.h
  19. 3
      table/plain_table_factory.cc
  20. 4
      table/plain_table_factory.h
  21. 3
      utilities/options/options_util_test.cc

@ -1210,6 +1210,37 @@ class PinL0IndexAndFilterBlocksTest : public DBTestBase,
PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {} PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {}
virtual void SetUp() override { infinite_max_files_ = GetParam(); } virtual void SetUp() override { infinite_max_files_ = GetParam(); }
void CreateTwoLevels(Options* options) {
if (infinite_max_files_) {
options->max_open_files = -1;
}
options->create_if_missing = true;
options->statistics = rocksdb::CreateDBStatistics();
BlockBasedTableOptions table_options;
table_options.cache_index_and_filter_blocks = true;
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
options->table_factory.reset(new BlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, *options);
Put(1, "a", "begin");
Put(1, "z", "end");
ASSERT_OK(Flush(1));
// move this table to L1
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
// reset block cache
table_options.block_cache = NewLRUCache(64 * 1024);
options->table_factory.reset(NewBlockBasedTableFactory(table_options));
TryReopenWithColumnFamilies({"default", "pikachu"}, *options);
// create new table at L0
Put(1, "a2", "begin2");
Put(1, "z2", "end2");
ASSERT_OK(Flush(1));
table_options.block_cache->EraseUnRefEntries();
}
bool infinite_max_files_; bool infinite_max_files_;
}; };
@ -1261,35 +1292,7 @@ TEST_P(PinL0IndexAndFilterBlocksTest,
TEST_P(PinL0IndexAndFilterBlocksTest, TEST_P(PinL0IndexAndFilterBlocksTest,
MultiLevelIndexAndFilterBlocksCachedWithPinning) { MultiLevelIndexAndFilterBlocksCachedWithPinning) {
Options options = CurrentOptions(); Options options = CurrentOptions();
if (infinite_max_files_) { PinL0IndexAndFilterBlocksTest::CreateTwoLevels(&options);
options.max_open_files = -1;
}
options.create_if_missing = true;
options.statistics = rocksdb::CreateDBStatistics();
BlockBasedTableOptions table_options;
table_options.cache_index_and_filter_blocks = true;
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
options.table_factory.reset(new BlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, options);
Put(1, "a", "begin");
Put(1, "z", "end");
ASSERT_OK(Flush(1));
// move this table to L1
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
// reset block cache
table_options.block_cache = NewLRUCache(64 * 1024);
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
TryReopenWithColumnFamilies({"default", "pikachu"}, options);
// create new table at L0
Put(1, "a2", "begin2");
Put(1, "z2", "end2");
ASSERT_OK(Flush(1));
table_options.block_cache->EraseUnRefEntries();
// get base cache values // get base cache values
uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS); uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS);
uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT); uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
@ -1309,11 +1312,76 @@ TEST_P(PinL0IndexAndFilterBlocksTest,
// the file is opened, prefetching results in a cache filter miss // the file is opened, prefetching results in a cache filter miss
// the block is loaded and added to the cache, // the block is loaded and added to the cache,
// then the get results in a cache hit for L1 // then the get results in a cache hit for L1
// When we have inifinite max_files, there is still cache miss because we have
// reset the block cache
value = Get(1, "a"); value = Get(1, "a");
ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
} }
TEST_P(PinL0IndexAndFilterBlocksTest, DisablePrefetchingNonL0IndexAndFilter) {
Options options = CurrentOptions();
PinL0IndexAndFilterBlocksTest::CreateTwoLevels(&options);
// Get base cache values
uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS);
uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT);
// Reopen database. If max_open_files is set as -1, table readers will be
// preloaded. This will trigger a BlockBasedTable::Open() and prefetch
// L0 index and filter. Level 1's prefetching is disabled in DB::Open()
TryReopenWithColumnFamilies({"default", "pikachu"}, options);
if (infinite_max_files_) {
// After reopen, cache miss are increased by one because we read (and only
// read) filter and index on L0
ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
} else {
// If max_open_files is not -1, we do not preload table readers, so there is
// no change.
ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
}
std::string value;
// this should be read from L0
value = Get(1, "a2");
// If max_open_files is -1, we have pinned index and filter in Rep, so there
// will not be changes in index and filter misses or hits. If max_open_files
// is not -1, Get() will open a TableReader and prefetch index and filter.
ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
// this should be read from L1
value = Get(1, "a");
if (infinite_max_files_) {
// In inifinite max files case, there's a cache miss in executing Get()
// because index and filter are not prefetched before.
ASSERT_EQ(fm + 2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
ASSERT_EQ(im + 2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
} else {
// In this case, cache miss will be increased by one in
// BlockBasedTable::Open() because this is not in DB::Open() code path so we
// will prefetch L1's index and filter. Cache hit will also be increased by
// one because Get() will read index and filter from the block cache
// prefetched in previous Open() call.
ASSERT_EQ(fm + 2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
ASSERT_EQ(fh + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
ASSERT_EQ(im + 2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
ASSERT_EQ(ih + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
}
}
INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest, INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest,
PinL0IndexAndFilterBlocksTest, ::testing::Bool()); PinL0IndexAndFilterBlocksTest, ::testing::Bool());
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE

@ -324,10 +324,11 @@ class TestPlainTableFactory : public PlainTableFactory {
column_family_id_(column_family_id), column_family_id_(column_family_id),
column_family_name_(std::move(column_family_name)) {} column_family_name_(std::move(column_family_name)) {}
Status NewTableReader(const TableReaderOptions& table_reader_options, Status NewTableReader(
unique_ptr<RandomAccessFileReader>&& file, const TableReaderOptions& table_reader_options,
uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table) const override { unique_ptr<TableReader>* table,
bool prefetch_index_and_filter_in_cache) const override {
TableProperties* props = nullptr; TableProperties* props = nullptr;
auto s = auto s =
ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber, ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,

@ -89,7 +89,7 @@ Status TableCache::GetTableReader(
const InternalKeyComparator& internal_comparator, const FileDescriptor& fd, const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
bool sequential_mode, size_t readahead, bool record_read_stats, bool sequential_mode, size_t readahead, bool record_read_stats,
HistogramImpl* file_read_hist, unique_ptr<TableReader>* table_reader, HistogramImpl* file_read_hist, unique_ptr<TableReader>* table_reader,
bool skip_filters, int level) { bool skip_filters, int level, bool prefetch_index_and_filter_in_cache) {
std::string fname = std::string fname =
TableFileName(ioptions_.db_paths, fd.GetNumber(), fd.GetPathId()); TableFileName(ioptions_.db_paths, fd.GetNumber(), fd.GetPathId());
unique_ptr<RandomAccessFile> file; unique_ptr<RandomAccessFile> file;
@ -111,7 +111,8 @@ Status TableCache::GetTableReader(
s = ioptions_.table_factory->NewTableReader( s = ioptions_.table_factory->NewTableReader(
TableReaderOptions(ioptions_, env_options, internal_comparator, TableReaderOptions(ioptions_, env_options, internal_comparator,
skip_filters, level), skip_filters, level),
std::move(file_reader), fd.GetFileSize(), table_reader); std::move(file_reader), fd.GetFileSize(), table_reader,
prefetch_index_and_filter_in_cache);
TEST_SYNC_POINT("TableCache::GetTableReader:0"); TEST_SYNC_POINT("TableCache::GetTableReader:0");
} }
return s; return s;
@ -129,7 +130,8 @@ Status TableCache::FindTable(const EnvOptions& env_options,
const FileDescriptor& fd, Cache::Handle** handle, const FileDescriptor& fd, Cache::Handle** handle,
const bool no_io, bool record_read_stats, const bool no_io, bool record_read_stats,
HistogramImpl* file_read_hist, bool skip_filters, HistogramImpl* file_read_hist, bool skip_filters,
int level) { int level,
bool prefetch_index_and_filter_in_cache) {
PERF_TIMER_GUARD(find_table_nanos); PERF_TIMER_GUARD(find_table_nanos);
Status s; Status s;
uint64_t number = fd.GetNumber(); uint64_t number = fd.GetNumber();
@ -146,7 +148,7 @@ Status TableCache::FindTable(const EnvOptions& env_options,
s = GetTableReader(env_options, internal_comparator, fd, s = GetTableReader(env_options, internal_comparator, fd,
false /* sequential mode */, 0 /* readahead */, false /* sequential mode */, 0 /* readahead */,
record_read_stats, file_read_hist, &table_reader, record_read_stats, file_read_hist, &table_reader,
skip_filters, level); skip_filters, level, prefetch_index_and_filter_in_cache);
if (!s.ok()) { if (!s.ok()) {
assert(table_reader == nullptr); assert(table_reader == nullptr);
RecordTick(ioptions_.statistics, NO_FILE_ERRORS); RecordTick(ioptions_.statistics, NO_FILE_ERRORS);

@ -79,7 +79,8 @@ class TableCache {
const FileDescriptor& file_fd, Cache::Handle**, const FileDescriptor& file_fd, Cache::Handle**,
const bool no_io = false, bool record_read_stats = true, const bool no_io = false, bool record_read_stats = true,
HistogramImpl* file_read_hist = nullptr, HistogramImpl* file_read_hist = nullptr,
bool skip_filters = false, int level = -1); bool skip_filters = false, int level = -1,
bool prefetch_index_and_filter_in_cache = true);
// Get TableReader from a cache handle. // Get TableReader from a cache handle.
TableReader* GetTableReaderFromHandle(Cache::Handle* handle); TableReader* GetTableReaderFromHandle(Cache::Handle* handle);
@ -114,7 +115,8 @@ class TableCache {
size_t readahead, bool record_read_stats, size_t readahead, bool record_read_stats,
HistogramImpl* file_read_hist, HistogramImpl* file_read_hist,
unique_ptr<TableReader>* table_reader, unique_ptr<TableReader>* table_reader,
bool skip_filters = false, int level = -1); bool skip_filters = false, int level = -1,
bool prefetch_index_and_filter_in_cache = true);
const ImmutableCFOptions& ioptions_; const ImmutableCFOptions& ioptions_;
const EnvOptions& env_options_; const EnvOptions& env_options_;

@ -287,7 +287,8 @@ class VersionBuilder::Rep {
CheckConsistency(vstorage); CheckConsistency(vstorage);
} }
void LoadTableHandlers(InternalStats* internal_stats, int max_threads) { void LoadTableHandlers(InternalStats* internal_stats, int max_threads,
bool prefetch_index_and_filter_in_cache) {
assert(table_cache_ != nullptr); assert(table_cache_ != nullptr);
// <file metadata, level> // <file metadata, level>
std::vector<std::pair<FileMetaData*, int>> files_meta; std::vector<std::pair<FileMetaData*, int>> files_meta;
@ -309,11 +310,12 @@ class VersionBuilder::Rep {
auto* file_meta = files_meta[file_idx].first; auto* file_meta = files_meta[file_idx].first;
int level = files_meta[file_idx].second; int level = files_meta[file_idx].second;
table_cache_->FindTable( table_cache_->FindTable(env_options_,
env_options_, *(base_vstorage_->InternalComparator()), *(base_vstorage_->InternalComparator()),
file_meta->fd, &file_meta->table_reader_handle, false /*no_io */, file_meta->fd, &file_meta->table_reader_handle,
true /* record_read_stats */, false /*no_io */, true /* record_read_stats */,
internal_stats->GetFileReadHist(level), false, level); internal_stats->GetFileReadHist(level), false,
level, prefetch_index_and_filter_in_cache);
if (file_meta->table_reader_handle != nullptr) { if (file_meta->table_reader_handle != nullptr) {
// Load table_reader // Load table_reader
file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle( file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
@ -363,9 +365,11 @@ void VersionBuilder::Apply(VersionEdit* edit) { rep_->Apply(edit); }
void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) { void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
rep_->SaveTo(vstorage); rep_->SaveTo(vstorage);
} }
void VersionBuilder::LoadTableHandlers(InternalStats* internal_stats, void VersionBuilder::LoadTableHandlers(
int max_threads) { InternalStats* internal_stats, int max_threads,
rep_->LoadTableHandlers(internal_stats, max_threads); bool prefetch_index_and_filter_in_cache) {
rep_->LoadTableHandlers(internal_stats, max_threads,
prefetch_index_and_filter_in_cache);
} }
void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level, void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level,
FileMetaData* f) { FileMetaData* f) {

@ -31,7 +31,8 @@ class VersionBuilder {
int level); int level);
void Apply(VersionEdit* edit); void Apply(VersionEdit* edit);
void SaveTo(VersionStorageInfo* vstorage); void SaveTo(VersionStorageInfo* vstorage);
void LoadTableHandlers(InternalStats* internal_stats, int max_threads = 1); void LoadTableHandlers(InternalStats* internal_stats, int max_threads,
bool prefetch_index_and_filter_in_cache);
void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f); void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f);
private: private:

@ -2272,7 +2272,8 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
// Need to do it out of the mutex. // Need to do it out of the mutex.
builder_guard->version_builder()->LoadTableHandlers( builder_guard->version_builder()->LoadTableHandlers(
column_family_data->internal_stats(), column_family_data->internal_stats(),
column_family_data->ioptions()->optimize_filters_for_hits); column_family_data->ioptions()->optimize_filters_for_hits,
true /* prefetch_index_and_filter_in_cache */);
} }
// This is fine because everything inside of this block is serialized -- // This is fine because everything inside of this block is serialized --
@ -2714,8 +2715,9 @@ Status VersionSet::Recover(
if (db_options_->max_open_files == -1) { if (db_options_->max_open_files == -1) {
// unlimited table cache. Pre-load table handle now. // unlimited table cache. Pre-load table handle now.
// Need to do it out of the mutex. // Need to do it out of the mutex.
builder->LoadTableHandlers(cfd->internal_stats(), builder->LoadTableHandlers(
db_options_->max_file_opening_threads); cfd->internal_stats(), db_options_->max_file_opening_threads,
false /* prefetch_index_and_filter_in_cache */);
} }
Version* v = new Version(cfd, this, current_version_number_++); Version* v = new Version(cfd, this, current_version_number_++);

@ -390,7 +390,8 @@ class TableFactory {
virtual Status NewTableReader( virtual Status NewTableReader(
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table_reader) const = 0; unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter_in_cache = true) const = 0;
// Return a table builder to write to a file for this table type. // Return a table builder to write to a file for this table type.
// //

@ -43,7 +43,8 @@ extern const uint64_t kCuckooTableMagicNumber;
Status AdaptiveTableFactory::NewTableReader( Status AdaptiveTableFactory::NewTableReader(
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table) const { unique_ptr<TableReader>* table,
bool prefetch_index_and_filter_in_cache) const {
Footer footer; Footer footer;
auto s = ReadFooterFromFile(file.get(), file_size, &footer); auto s = ReadFooterFromFile(file.get(), file_size, &footer);
if (!s.ok()) { if (!s.ok()) {

@ -33,10 +33,11 @@ class AdaptiveTableFactory : public TableFactory {
const char* Name() const override { return "AdaptiveTableFactory"; } const char* Name() const override { return "AdaptiveTableFactory"; }
Status NewTableReader(const TableReaderOptions& table_reader_options, Status NewTableReader(
unique_ptr<RandomAccessFileReader>&& file, const TableReaderOptions& table_reader_options,
uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table) const override; unique_ptr<TableReader>* table,
bool prefetch_index_and_filter_in_cache = true) const override;
TableBuilder* NewTableBuilder( TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,

@ -50,20 +50,12 @@ BlockBasedTableFactory::BlockBasedTableFactory(
Status BlockBasedTableFactory::NewTableReader( Status BlockBasedTableFactory::NewTableReader(
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table_reader) const { unique_ptr<TableReader>* table_reader,
return NewTableReader(table_reader_options, std::move(file), file_size, bool prefetch_index_and_filter_in_cache) const {
table_reader,
/*prefetch_index_and_filter=*/true);
}
Status BlockBasedTableFactory::NewTableReader(
const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table_reader, const bool prefetch_enabled) const {
return BlockBasedTable::Open( return BlockBasedTable::Open(
table_reader_options.ioptions, table_reader_options.env_options, table_reader_options.ioptions, table_reader_options.env_options,
table_options_, table_reader_options.internal_comparator, std::move(file), table_options_, table_reader_options.internal_comparator, std::move(file),
file_size, table_reader, prefetch_enabled, file_size, table_reader, prefetch_index_and_filter_in_cache,
table_reader_options.skip_filters, table_reader_options.level); table_reader_options.skip_filters, table_reader_options.level);
} }

@ -33,18 +33,11 @@ class BlockBasedTableFactory : public TableFactory {
const char* Name() const override { return "BlockBasedTable"; } const char* Name() const override { return "BlockBasedTable"; }
Status NewTableReader(const TableReaderOptions& table_reader_options, Status NewTableReader(
unique_ptr<RandomAccessFileReader>&& file, const TableReaderOptions& table_reader_options,
uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table_reader) const override; unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter_in_cache = true) const override;
// This is a variant of virtual member function NewTableReader function with
// added capability to disable pre-fetching of blocks on BlockBasedTable::Open
Status NewTableReader(const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file,
uint64_t file_size,
unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter) const;
TableBuilder* NewTableBuilder( TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,

@ -515,7 +515,7 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
unique_ptr<RandomAccessFileReader>&& file, unique_ptr<RandomAccessFileReader>&& file,
uint64_t file_size, uint64_t file_size,
unique_ptr<TableReader>* table_reader, unique_ptr<TableReader>* table_reader,
const bool prefetch_index_and_filter, const bool prefetch_index_and_filter_in_cache,
const bool skip_filters, const int level) { const bool skip_filters, const int level) {
table_reader->reset(); table_reader->reset();
@ -636,10 +636,11 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
BlockBasedTablePropertyNames::kPrefixFiltering, rep->ioptions.info_log); BlockBasedTablePropertyNames::kPrefixFiltering, rep->ioptions.info_log);
} }
if (prefetch_index_and_filter) {
// pre-fetching of blocks is turned on // pre-fetching of blocks is turned on
// Will use block cache for index/filter blocks access? // Will use block cache for index/filter blocks access
if (table_options.cache_index_and_filter_blocks) { // Always prefetch index and filter for level 0
if (table_options.cache_index_and_filter_blocks) {
if (prefetch_index_and_filter_in_cache || level == 0) {
assert(table_options.block_cache != nullptr); assert(table_options.block_cache != nullptr);
// Hack: Call NewIndexIterator() to implicitly add index to the // Hack: Call NewIndexIterator() to implicitly add index to the
// block_cache // block_cache
@ -663,7 +664,7 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
// if pin_l0_filter_and_index_blocks_in_cache is true, and this is // if pin_l0_filter_and_index_blocks_in_cache is true, and this is
// a level0 file, then save it in rep_->filter_entry; it will be // a level0 file, then save it in rep_->filter_entry; it will be
// released in the destructor only, hence it will be pinned in the // released in the destructor only, hence it will be pinned in the
// cache until this reader is alive // cache while this reader is alive
if (rep->table_options.pin_l0_filter_and_index_blocks_in_cache && if (rep->table_options.pin_l0_filter_and_index_blocks_in_cache &&
level == 0) { level == 0) {
rep->filter_entry = filter_entry; rep->filter_entry = filter_entry;
@ -671,23 +672,23 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
filter_entry.Release(table_options.block_cache.get()); filter_entry.Release(table_options.block_cache.get());
} }
} }
} else { }
// If we don't use block cache for index/filter blocks access, we'll } else {
// pre-load these blocks, which will kept in member variables in Rep // If we don't use block cache for index/filter blocks access, we'll
// and with a same life-time as this table object. // pre-load these blocks, which will kept in member variables in Rep
IndexReader* index_reader = nullptr; // and with a same life-time as this table object.
s = new_table->CreateIndexReader(&index_reader, meta_iter.get()); IndexReader* index_reader = nullptr;
s = new_table->CreateIndexReader(&index_reader, meta_iter.get());
if (s.ok()) { if (s.ok()) {
rep->index_reader.reset(index_reader); rep->index_reader.reset(index_reader);
// Set filter block // Set filter block
if (rep->filter_policy) { if (rep->filter_policy) {
rep->filter.reset(ReadFilter(rep)); rep->filter.reset(ReadFilter(rep));
}
} else {
delete index_reader;
} }
} else {
delete index_reader;
} }
} }

@ -69,17 +69,19 @@ class BlockBasedTable : public TableReader {
// to nullptr and returns a non-ok status. // to nullptr and returns a non-ok status.
// //
// @param file must remain live while this Table is in use. // @param file must remain live while this Table is in use.
// @param prefetch_index_and_filter can be used to disable prefetching of // @param prefetch_index_and_filter_in_cache can be used to disable
// index and filter blocks at startup // prefetching of
// index and filter blocks into block cache at startup
// @param skip_filters Disables loading/accessing the filter block. Overrides // @param skip_filters Disables loading/accessing the filter block. Overrides
// prefetch_index_and_filter, so filter will be skipped if both are set. // prefetch_index_and_filter_in_cache, so filter will be skipped if both
// are set.
static Status Open(const ImmutableCFOptions& ioptions, static Status Open(const ImmutableCFOptions& ioptions,
const EnvOptions& env_options, const EnvOptions& env_options,
const BlockBasedTableOptions& table_options, const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_key_comparator, const InternalKeyComparator& internal_key_comparator,
unique_ptr<RandomAccessFileReader>&& file, unique_ptr<RandomAccessFileReader>&& file,
uint64_t file_size, unique_ptr<TableReader>* table_reader, uint64_t file_size, unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter = true, bool prefetch_index_and_filter_in_cache = true,
bool skip_filters = false, int level = -1); bool skip_filters = false, int level = -1);
bool PrefixMayMatch(const Slice& internal_key); bool PrefixMayMatch(const Slice& internal_key);

@ -15,7 +15,8 @@ namespace rocksdb {
Status CuckooTableFactory::NewTableReader( Status CuckooTableFactory::NewTableReader(
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
std::unique_ptr<TableReader>* table) const { std::unique_ptr<TableReader>* table,
bool prefetch_index_and_filter_in_cache) const {
std::unique_ptr<CuckooTableReader> new_reader(new CuckooTableReader( std::unique_ptr<CuckooTableReader> new_reader(new CuckooTableReader(
table_reader_options.ioptions, std::move(file), file_size, table_reader_options.ioptions, std::move(file), file_size,
table_reader_options.internal_comparator.user_comparator(), nullptr)); table_reader_options.internal_comparator.user_comparator(), nullptr));

@ -55,10 +55,11 @@ class CuckooTableFactory : public TableFactory {
const char* Name() const override { return "CuckooTable"; } const char* Name() const override { return "CuckooTable"; }
Status NewTableReader(const TableReaderOptions& table_reader_options, Status NewTableReader(
unique_ptr<RandomAccessFileReader>&& file, const TableReaderOptions& table_reader_options,
uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table) const override; unique_ptr<TableReader>* table,
bool prefetch_index_and_filter_in_cache = true) const override;
TableBuilder* NewTableBuilder( TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,

@ -59,7 +59,8 @@ MockTableFactory::MockTableFactory() : next_id_(1) {}
Status MockTableFactory::NewTableReader( Status MockTableFactory::NewTableReader(
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table_reader) const { unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter_in_cache) const {
uint32_t id = GetIDFromFile(file.get()); uint32_t id = GetIDFromFile(file.get());
MutexLock lock_guard(&file_system_.mutex); MutexLock lock_guard(&file_system_.mutex);

@ -147,10 +147,11 @@ class MockTableFactory : public TableFactory {
public: public:
MockTableFactory(); MockTableFactory();
const char* Name() const override { return "MockTable"; } const char* Name() const override { return "MockTable"; }
Status NewTableReader(const TableReaderOptions& table_reader_options, Status NewTableReader(
unique_ptr<RandomAccessFileReader>&& file, const TableReaderOptions& table_reader_options,
uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table_reader) const override; unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter_in_cache = true) const override;
TableBuilder* NewTableBuilder( TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,
uint32_t column_familly_id, WritableFileWriter* file) const override; uint32_t column_familly_id, WritableFileWriter* file) const override;

@ -17,7 +17,8 @@ namespace rocksdb {
Status PlainTableFactory::NewTableReader( Status PlainTableFactory::NewTableReader(
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table) const { unique_ptr<TableReader>* table,
bool prefetch_index_and_filter_in_cache) const {
return PlainTableReader::Open( return PlainTableReader::Open(
table_reader_options.ioptions, table_reader_options.env_options, table_reader_options.ioptions, table_reader_options.env_options,
table_reader_options.internal_comparator, std::move(file), file_size, table_reader_options.internal_comparator, std::move(file), file_size,

@ -149,8 +149,8 @@ class PlainTableFactory : public TableFactory {
const char* Name() const override { return "PlainTable"; } const char* Name() const override { return "PlainTable"; }
Status NewTableReader(const TableReaderOptions& table_reader_options, Status NewTableReader(const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, unique_ptr<RandomAccessFileReader>&& file,
uint64_t file_size, uint64_t file_size, unique_ptr<TableReader>* table,
unique_ptr<TableReader>* table) const override; bool prefetch_index_and_filter_in_cache) const override;
TableBuilder* NewTableBuilder( TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,

@ -105,7 +105,8 @@ class DummyTableFactory : public TableFactory {
virtual Status NewTableReader(const TableReaderOptions& table_reader_options, virtual Status NewTableReader(const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file, unique_ptr<RandomAccessFileReader>&& file,
uint64_t file_size, uint64_t file_size,
unique_ptr<TableReader>* table_reader) const { unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter_in_cache) const {
return Status::NotSupported(); return Status::NotSupported();
} }

Loading…
Cancel
Save