NewLRUCache() to pick number of shard bits based on capacity if not given

Summary:
If the users use the NewLRUCache() without passing in the number of shard bits, instead of using hard-coded 6, we'll determine it based on capacity.
Closes https://github.com/facebook/rocksdb/pull/1584

Differential Revision: D4242517

Pulled By: siying

fbshipit-source-id: 86b0f18
main
Siying Dong 8 years ago committed by Facebook Github Bot
parent f25f1ec60b
commit 2d75cd40d3
  1. 1
      HISTORY.md
  2. 5
      db/corruption_test.cc
  3. 6
      include/rocksdb/cache.h
  4. 28
      util/cache_test.cc
  5. 3
      util/clock_cache.cc
  6. 3
      util/lru_cache.cc
  7. 12
      util/sharded_cache.cc
  8. 4
      util/sharded_cache.h

@ -5,6 +5,7 @@
* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully. * Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env. * BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations. * Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
### Bug Fixes ### Bug Fixes
* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions. * Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.

@ -41,7 +41,10 @@ class CorruptionTest : public testing::Test {
DB* db_; DB* db_;
CorruptionTest() { CorruptionTest() {
tiny_cache_ = NewLRUCache(100); // If LRU cache shard bit is smaller than 2 (or -1 which will automatically
// set it to 0), test SequenceNumberRecovery will fail, likely because of a
// bug in recovery code. Keep it 4 for now to make the test passes.
tiny_cache_ = NewLRUCache(100, 2);
options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
options_.env = &env_; options_.env = &env_;
dbname_ = test::TmpDir() + "/corruption_test"; dbname_ = test::TmpDir() + "/corruption_test";

@ -39,8 +39,10 @@ class Cache;
// is set, insert to the cache will fail when cache is full. User can also // is set, insert to the cache will fail when cache is full. User can also
// set percentage of the cache reserves for high priority entries via // set percentage of the cache reserves for high priority entries via
// high_pri_pool_pct. // high_pri_pool_pct.
// num_shard_bits = -1 means it is automatically determined: every shard
// will be at least 512KB and number of shard bits will not exceed 6.
extern std::shared_ptr<Cache> NewLRUCache(size_t capacity, extern std::shared_ptr<Cache> NewLRUCache(size_t capacity,
int num_shard_bits = 6, int num_shard_bits = -1,
bool strict_capacity_limit = false, bool strict_capacity_limit = false,
double high_pri_pool_ratio = 0.0); double high_pri_pool_ratio = 0.0);
@ -50,7 +52,7 @@ extern std::shared_ptr<Cache> NewLRUCache(size_t capacity,
// //
// Return nullptr if it is not supported. // Return nullptr if it is not supported.
extern std::shared_ptr<Cache> NewClockCache(size_t capacity, extern std::shared_ptr<Cache> NewClockCache(size_t capacity,
int num_shard_bits = 6, int num_shard_bits = -1,
bool strict_capacity_limit = false); bool strict_capacity_limit = false);
class Cache { class Cache {

@ -16,6 +16,7 @@
#include <vector> #include <vector>
#include "util/clock_cache.h" #include "util/clock_cache.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/lru_cache.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "util/testharness.h" #include "util/testharness.h"
@ -75,6 +76,17 @@ class CacheTest : public testing::TestWithParam<std::string> {
~CacheTest() { ~CacheTest() {
} }
std::shared_ptr<Cache> NewCache(size_t capacity) {
auto type = GetParam();
if (type == kLRU) {
return NewLRUCache(capacity);
}
if (type == kClock) {
return NewClockCache(capacity);
}
return nullptr;
}
std::shared_ptr<Cache> NewCache(size_t capacity, int num_shard_bits, std::shared_ptr<Cache> NewCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit) { bool strict_capacity_limit) {
auto type = GetParam(); auto type = GetParam();
@ -628,6 +640,22 @@ TEST_P(CacheTest, ApplyToAllCacheEntiresTest) {
ASSERT_TRUE(inserted == callback_state); ASSERT_TRUE(inserted == callback_state);
} }
TEST_P(CacheTest, DefaultShardBits) {
// test1: set the flag to false. Insert more keys than capacity. See if they
// all go through.
std::shared_ptr<Cache> cache = NewCache(16 * 1024L * 1024L);
ShardedCache* sc = dynamic_cast<ShardedCache*>(cache.get());
ASSERT_EQ(5, sc->GetNumShardBits());
cache = NewLRUCache(511 * 1024L, -1, true);
sc = dynamic_cast<ShardedCache*>(cache.get());
ASSERT_EQ(0, sc->GetNumShardBits());
cache = NewLRUCache(1024L * 1024L * 1024L, -1, true);
sc = dynamic_cast<ShardedCache*>(cache.get());
ASSERT_EQ(6, sc->GetNumShardBits());
}
#ifdef SUPPORT_CLOCK_CACHE #ifdef SUPPORT_CLOCK_CACHE
shared_ptr<Cache> (*new_clock_cache_func)(size_t, int, bool) = NewClockCache; shared_ptr<Cache> (*new_clock_cache_func)(size_t, int, bool) = NewClockCache;
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest, INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,

@ -695,6 +695,9 @@ class ClockCache : public ShardedCache {
std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits, std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit) { bool strict_capacity_limit) {
if (num_shard_bits < 0) {
num_shard_bits = GetDefaultCacheShardBits(capacity);
}
return std::make_shared<ClockCache>(capacity, num_shard_bits, return std::make_shared<ClockCache>(capacity, num_shard_bits,
strict_capacity_limit); strict_capacity_limit);
} }

@ -480,6 +480,9 @@ std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
// invalid high_pri_pool_ratio // invalid high_pri_pool_ratio
return nullptr; return nullptr;
} }
if (num_shard_bits < 0) {
num_shard_bits = GetDefaultCacheShardBits(capacity);
}
return std::make_shared<LRUCache>(capacity, num_shard_bits, return std::make_shared<LRUCache>(capacity, num_shard_bits,
strict_capacity_limit, high_pri_pool_ratio); strict_capacity_limit, high_pri_pool_ratio);
} }

@ -145,5 +145,17 @@ std::string ShardedCache::GetPrintableOptions() const {
ret.append(GetShard(0)->GetPrintableOptions()); ret.append(GetShard(0)->GetPrintableOptions());
return ret; return ret;
} }
int GetDefaultCacheShardBits(size_t capacity) {
int num_shard_bits = 0;
size_t min_shard_size = 512L * 1024L; // Every shard is at least 512KB.
size_t num_shards = capacity / min_shard_size;
while (num_shards >>= 1) {
if (++num_shard_bits >= 6) {
// No more than 6.
return num_shard_bits;
}
}
return num_shard_bits;
}
} // namespace rocksdb } // namespace rocksdb

@ -78,6 +78,8 @@ class ShardedCache : public Cache {
virtual void EraseUnRefEntries() override; virtual void EraseUnRefEntries() override;
virtual std::string GetPrintableOptions() const override; virtual std::string GetPrintableOptions() const override;
int GetNumShardBits() const { return num_shard_bits_; }
private: private:
static inline uint32_t HashSlice(const Slice& s) { static inline uint32_t HashSlice(const Slice& s) {
return Hash(s.data(), s.size(), 0); return Hash(s.data(), s.size(), 0);
@ -95,4 +97,6 @@ class ShardedCache : public Cache {
std::atomic<uint64_t> last_id_; std::atomic<uint64_t> last_id_;
}; };
extern int GetDefaultCacheShardBits(size_t capacity);
} // namespace rocksdb } // namespace rocksdb

Loading…
Cancel
Save