Summary: Added a new abstraction to cache page to RocksDB designed for the read cache use. RocksDB current block cache is more of an object cache. For the persistent read cache project, what we need is a page cache equivalent. This changes adds a cache abstraction to RocksDB to cache pages called PersistentCache. PersistentCache can cache uncompressed pages or raw pages (content as in filesystem). The user can choose to operate PersistentCache either in COMPRESSED or UNCOMPRESSED mode. Blame Rev: Test Plan: Run unit tests Reviewers: sdong Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D55707main
krad
9 years ago
parent
5c06e0814c
commit
a08c8c851a
@ -0,0 +1,49 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
#include <memory> |
||||||
|
|
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/statistics.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
// PersistentCache
|
||||||
|
//
|
||||||
|
// Persistent cache interface for caching IO pages on a persistent medium. The
|
||||||
|
// cache interface is specifically designed for persistent read cache.
|
||||||
|
class PersistentCache { |
||||||
|
public: |
||||||
|
virtual ~PersistentCache() {} |
||||||
|
|
||||||
|
// Insert to page cache
|
||||||
|
//
|
||||||
|
// page_key Identifier to identify a page uniquely across restarts
|
||||||
|
// data Page data
|
||||||
|
// size Size of the page
|
||||||
|
virtual Status Insert(const Slice& key, const char* data, |
||||||
|
const size_t size) = 0; |
||||||
|
|
||||||
|
// Lookup page cache by page identifier
|
||||||
|
//
|
||||||
|
// page_key Page identifier
|
||||||
|
// buf Buffer where the data should be copied
|
||||||
|
// size Size of the page
|
||||||
|
virtual Status Lookup(const Slice& key, std::unique_ptr<char[]>* data, |
||||||
|
size_t* size) = 0; |
||||||
|
|
||||||
|
// Is cache storing uncompressed data ?
|
||||||
|
//
|
||||||
|
// True if the cache is configured to store uncompressed data else false
|
||||||
|
virtual bool IsCompressed() = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,112 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#include "table/persistent_cache_helper.h" |
||||||
|
#include "table/format.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
void PersistentCacheHelper::InsertRawPage( |
||||||
|
const PersistentCacheOptions& cache_options, const BlockHandle& handle, |
||||||
|
const char* data, const size_t size) { |
||||||
|
assert(cache_options.persistent_cache); |
||||||
|
assert(cache_options.persistent_cache->IsCompressed()); |
||||||
|
|
||||||
|
// construct the page key
|
||||||
|
char cache_key[BlockBasedTable::kMaxCacheKeyPrefixSize + kMaxVarint64Length]; |
||||||
|
auto key = BlockBasedTable::GetCacheKey(cache_options.key_prefix.c_str(), |
||||||
|
cache_options.key_prefix.size(), |
||||||
|
handle, cache_key); |
||||||
|
// insert content to cache
|
||||||
|
cache_options.persistent_cache->Insert(key, data, size); |
||||||
|
} |
||||||
|
|
||||||
|
void PersistentCacheHelper::InsertUncompressedPage( |
||||||
|
const PersistentCacheOptions& cache_options, const BlockHandle& handle, |
||||||
|
const BlockContents& contents) { |
||||||
|
assert(cache_options.persistent_cache); |
||||||
|
assert(!cache_options.persistent_cache->IsCompressed()); |
||||||
|
if (!contents.cachable || contents.compression_type != kNoCompression) { |
||||||
|
// We shouldn't cache this. Either
|
||||||
|
// (1) content is not cacheable
|
||||||
|
// (2) content is compressed
|
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
// construct the page key
|
||||||
|
char cache_key[BlockBasedTable::kMaxCacheKeyPrefixSize + kMaxVarint64Length]; |
||||||
|
auto key = BlockBasedTable::GetCacheKey(cache_options.key_prefix.c_str(), |
||||||
|
cache_options.key_prefix.size(), |
||||||
|
handle, cache_key); |
||||||
|
// insert block contents to page cache
|
||||||
|
cache_options.persistent_cache->Insert(key, contents.data.data(), |
||||||
|
contents.data.size()); |
||||||
|
} |
||||||
|
|
||||||
|
Status PersistentCacheHelper::LookupRawPage( |
||||||
|
const PersistentCacheOptions& cache_options, const BlockHandle& handle, |
||||||
|
std::unique_ptr<char[]>* raw_data, const size_t raw_data_size) { |
||||||
|
assert(cache_options.persistent_cache); |
||||||
|
assert(cache_options.persistent_cache->IsCompressed()); |
||||||
|
|
||||||
|
// construct the page key
|
||||||
|
char cache_key[BlockBasedTable::kMaxCacheKeyPrefixSize + kMaxVarint64Length]; |
||||||
|
auto key = BlockBasedTable::GetCacheKey(cache_options.key_prefix.c_str(), |
||||||
|
cache_options.key_prefix.size(), |
||||||
|
handle, cache_key); |
||||||
|
// Lookup page
|
||||||
|
size_t size; |
||||||
|
Status s = cache_options.persistent_cache->Lookup(key, raw_data, &size); |
||||||
|
if (!s.ok()) { |
||||||
|
// cache miss
|
||||||
|
RecordTick(cache_options.statistics, PERSISTENT_CACHE_MISS); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
// cache hit
|
||||||
|
assert(raw_data_size == handle.size() + kBlockTrailerSize); |
||||||
|
assert(size == raw_data_size); |
||||||
|
RecordTick(cache_options.statistics, PERSISTENT_CACHE_HIT); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status PersistentCacheHelper::LookupUncompressedPage( |
||||||
|
const PersistentCacheOptions& cache_options, const BlockHandle& handle, |
||||||
|
BlockContents* contents) { |
||||||
|
assert(cache_options.persistent_cache); |
||||||
|
assert(!cache_options.persistent_cache->IsCompressed()); |
||||||
|
if (!contents) { |
||||||
|
// We shouldn't lookup in the cache. Either
|
||||||
|
// (1) Nowhere to store
|
||||||
|
return Status::NotFound(); |
||||||
|
} |
||||||
|
|
||||||
|
// construct the page key
|
||||||
|
char cache_key[BlockBasedTable::kMaxCacheKeyPrefixSize + kMaxVarint64Length]; |
||||||
|
auto key = BlockBasedTable::GetCacheKey(cache_options.key_prefix.c_str(), |
||||||
|
cache_options.key_prefix.size(), |
||||||
|
handle, cache_key); |
||||||
|
// Lookup page
|
||||||
|
std::unique_ptr<char[]> data; |
||||||
|
size_t size; |
||||||
|
Status s = cache_options.persistent_cache->Lookup(key, &data, &size); |
||||||
|
if (!s.ok()) { |
||||||
|
// cache miss
|
||||||
|
RecordTick(cache_options.statistics, PERSISTENT_CACHE_MISS); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
// please note we are potentially comparing compressed data size with
|
||||||
|
// uncompressed data size
|
||||||
|
assert(handle.size() <= size); |
||||||
|
|
||||||
|
// update stats
|
||||||
|
RecordTick(cache_options.statistics, PERSISTENT_CACHE_HIT); |
||||||
|
// construct result and return
|
||||||
|
*contents = |
||||||
|
BlockContents(std::move(data), size, false /*cacheable*/, kNoCompression); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,63 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "table/block_based_table_reader.h" |
||||||
|
#include "util/statistics.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
struct BlockContents; |
||||||
|
|
||||||
|
// PersistentCacheOptions
|
||||||
|
//
|
||||||
|
// This describe the caching behavior for page cache
|
||||||
|
// This is used to pass the context for caching and the cache handle
|
||||||
|
struct PersistentCacheOptions { |
||||||
|
PersistentCacheOptions() {} |
||||||
|
explicit PersistentCacheOptions( |
||||||
|
const std::shared_ptr<PersistentCache>& _persistent_cache, |
||||||
|
const std::string _key_prefix, Statistics* const _statistics) |
||||||
|
: persistent_cache(_persistent_cache), |
||||||
|
key_prefix(_key_prefix), |
||||||
|
statistics(_statistics) {} |
||||||
|
|
||||||
|
virtual ~PersistentCacheOptions() {} |
||||||
|
|
||||||
|
std::shared_ptr<PersistentCache> persistent_cache; |
||||||
|
std::string key_prefix; |
||||||
|
Statistics* statistics = nullptr; |
||||||
|
}; |
||||||
|
|
||||||
|
// PersistentCacheHelper
|
||||||
|
//
|
||||||
|
// Encapsulates some of the helper logic for read and writing from the cache
|
||||||
|
class PersistentCacheHelper { |
||||||
|
public: |
||||||
|
// insert block into raw page cache
|
||||||
|
static void InsertRawPage(const PersistentCacheOptions& cache_options, |
||||||
|
const BlockHandle& handle, const char* data, |
||||||
|
const size_t size); |
||||||
|
|
||||||
|
// insert block into uncompressed cache
|
||||||
|
static void InsertUncompressedPage( |
||||||
|
const PersistentCacheOptions& cache_options, const BlockHandle& handle, |
||||||
|
const BlockContents& contents); |
||||||
|
|
||||||
|
// lookup block from raw page cacge
|
||||||
|
static Status LookupRawPage(const PersistentCacheOptions& cache_options, |
||||||
|
const BlockHandle& handle, |
||||||
|
std::unique_ptr<char[]>* raw_data, |
||||||
|
const size_t raw_data_size); |
||||||
|
|
||||||
|
// lookup block from uncompressed cache
|
||||||
|
static Status LookupUncompressedPage( |
||||||
|
const PersistentCacheOptions& cache_options, const BlockHandle& handle, |
||||||
|
BlockContents* contents); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
Loading…
Reference in new issue