From 2914de64e8b3556013b081cf9d745fb432c7ce54 Mon Sep 17 00:00:00 2001 From: Aaron Gao Date: Wed, 10 Aug 2016 17:42:24 -0700 Subject: [PATCH] add sim_cache stats to Statistics Summary: add SIM_BLOCK_CACHE_HIT and SIM_BLOCK_CACHE_MISS tickers. maybe can be combined with Histograms like DB_GET to evaluate the current setting of the size of block cache. Test Plan: make all check Reviewers: sdong, andrewkr, IslamAbdelRahman, yiwu Reviewed By: yiwu Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D61803 --- include/rocksdb/statistics.h | 7 ++++ include/rocksdb/utilities/sim_cache.h | 11 +++--- utilities/simulator_cache/sim_cache.cc | 42 ++++++++++++--------- utilities/simulator_cache/sim_cache_test.cc | 6 ++- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index 16fba0906..bb7134efe 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -68,6 +68,11 @@ enum Tickers : uint32_t { // # persistent cache miss PERSISTENT_CACHE_MISS, + // # total simulation block cache hits + SIM_BLOCK_CACHE_HIT, + // # total simulation block cache misses + SIM_BLOCK_CACHE_MISS, + // # of memtable hits. MEMTABLE_HIT, // # of memtable misses. @@ -218,6 +223,8 @@ const std::vector> TickersNameMap = { {BLOOM_FILTER_USEFUL, "rocksdb.bloom.filter.useful"}, {PERSISTENT_CACHE_HIT, "rocksdb.persistent.cache.hit"}, {PERSISTENT_CACHE_MISS, "rocksdb.persistent.cache.miss"}, + {SIM_BLOCK_CACHE_HIT, "rocksdb.sim.block.cache.hit"}, + {SIM_BLOCK_CACHE_MISS, "rocksdb.sim.block.cache.miss"}, {MEMTABLE_HIT, "rocksdb.memtable.hit"}, {MEMTABLE_MISS, "rocksdb.memtable.miss"}, {GET_HIT_L0, "rocksdb.l0.hit"}, diff --git a/include/rocksdb/utilities/sim_cache.h b/include/rocksdb/utilities/sim_cache.h index cc8a01bec..d6f31cd8e 100644 --- a/include/rocksdb/utilities/sim_cache.h +++ b/include/rocksdb/utilities/sim_cache.h @@ -11,6 +11,7 @@ #include "rocksdb/cache.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" +#include "util/statistics.h" namespace rocksdb { @@ -22,9 +23,9 @@ class SimCache; // to predict block cache hit rate without actually allocating the memory. It // can help users tune their current block cache size, and determine how // efficient they are using the memory. -extern std::shared_ptr NewSimCache(std::shared_ptr cache, - size_t sim_capacity, - int num_shard_bits); +extern std::shared_ptr NewSimCache( + std::shared_ptr cache, size_t sim_capacity, int num_shard_bits, + std::shared_ptr stats = nullptr); class SimCache : public Cache { public: @@ -48,11 +49,9 @@ class SimCache : public Cache { virtual void SetSimCapacity(size_t capacity) = 0; // returns the lookup times of simcache - virtual uint64_t get_lookup_counter() const = 0; + virtual uint64_t get_miss_counter() const = 0; // returns the hit times of simcache virtual uint64_t get_hit_counter() const = 0; - // returns the hit rate of simcache - virtual double get_hit_rate() const = 0; // reset the lookup and hit counters virtual void reset_counter() = 0; // String representation of the statistics of the simcache diff --git a/utilities/simulator_cache/sim_cache.cc b/utilities/simulator_cache/sim_cache.cc index 1579a33da..0ac8798a2 100644 --- a/utilities/simulator_cache/sim_cache.cc +++ b/utilities/simulator_cache/sim_cache.cc @@ -16,11 +16,12 @@ class SimCacheImpl : public SimCache { // capacity for real cache (ShardedLRUCache) // test_capacity for key only cache SimCacheImpl(std::shared_ptr cache, size_t sim_capacity, - int num_shard_bits) + int num_shard_bits, Statistics* stats) : cache_(cache), key_only_cache_(NewLRUCache(sim_capacity, num_shard_bits)), - lookup_times_(0), - hit_times_(0) {} + miss_times_(0), + hit_times_(0), + stats_(stats) {} virtual ~SimCacheImpl() {} virtual void SetCapacity(size_t capacity) override { @@ -50,11 +51,14 @@ class SimCacheImpl : public SimCache { } virtual Handle* Lookup(const Slice& key) override { - inc_lookup_counter(); Handle* h = key_only_cache_->Lookup(key); if (h != nullptr) { key_only_cache_->Release(h); inc_hit_counter(); + RecordTick(stats_, SIM_BLOCK_CACHE_HIT); + } else { + inc_miss_counter(); + RecordTick(stats_, SIM_BLOCK_CACHE_MISS); } return cache_->Lookup(key); } @@ -112,30 +116,29 @@ class SimCacheImpl : public SimCache { key_only_cache_->SetCapacity(capacity); } - virtual uint64_t get_lookup_counter() const override { - return lookup_times_.load(std::memory_order_relaxed); + virtual uint64_t get_miss_counter() const override { + return miss_times_.load(std::memory_order_relaxed); } virtual uint64_t get_hit_counter() const override { return hit_times_.load(std::memory_order_relaxed); } - virtual double get_hit_rate() const override { - return get_hit_counter() * 1.0f / get_lookup_counter(); - } virtual void reset_counter() override { - lookup_times_.store(0, std::memory_order_relaxed); + miss_times_.store(0, std::memory_order_relaxed); hit_times_.store(0, std::memory_order_relaxed); + SetTickerCount(stats_, SIM_BLOCK_CACHE_HIT, 0); + SetTickerCount(stats_, SIM_BLOCK_CACHE_MISS, 0); } virtual std::string ToString() const override { std::string res; - res.append("SimCache LOOKUPs: " + std::to_string(get_lookup_counter()) + - "\n"); + res.append("SimCache MISSes: " + std::to_string(get_miss_counter()) + "\n"); res.append("SimCache HITs: " + std::to_string(get_hit_counter()) + "\n"); char buff[100]; + auto lookups = get_miss_counter() + get_hit_counter(); snprintf(buff, sizeof(buff), "SimCache HITRATE: %.2f%%\n", - get_hit_rate() * 100); + (lookups == 0 ? 0 : get_hit_counter() * 100.0f / lookups)); res.append(buff); return res; } @@ -143,10 +146,11 @@ class SimCacheImpl : public SimCache { private: std::shared_ptr cache_; std::shared_ptr key_only_cache_; - std::atomic lookup_times_; + std::atomic miss_times_; std::atomic hit_times_; - void inc_lookup_counter() { - lookup_times_.fetch_add(1, std::memory_order_relaxed); + Statistics* stats_; + void inc_miss_counter() { + miss_times_.fetch_add(1, std::memory_order_relaxed); } void inc_hit_counter() { hit_times_.fetch_add(1, std::memory_order_relaxed); } }; @@ -155,11 +159,13 @@ class SimCacheImpl : public SimCache { // For instrumentation purpose, use NewSimCache instead std::shared_ptr NewSimCache(std::shared_ptr cache, - size_t sim_capacity, int num_shard_bits) { + size_t sim_capacity, int num_shard_bits, + std::shared_ptr stats) { if (num_shard_bits >= 20) { return nullptr; // the cache cannot be sharded into too many fine pieces } - return std::make_shared(cache, sim_capacity, num_shard_bits); + return std::make_shared(cache, sim_capacity, num_shard_bits, + stats.get()); } } // end namespace rocksdb diff --git a/utilities/simulator_cache/sim_cache_test.cc b/utilities/simulator_cache/sim_cache_test.cc index 419a3ddbb..4d8c3a7de 100644 --- a/utilities/simulator_cache/sim_cache_test.cc +++ b/utilities/simulator_cache/sim_cache_test.cc @@ -95,7 +95,8 @@ TEST_F(SimCacheTest, SimCache) { CheckCacheCounters(options, 1, 0, 1, 0); iterators[i].reset(iter); } - ASSERT_EQ(kNumBlocks, simCache->get_lookup_counter()); + ASSERT_EQ(kNumBlocks, + simCache->get_hit_counter() + simCache->get_miss_counter()); ASSERT_EQ(0, simCache->get_hit_counter()); size_t usage = simCache->GetUsage(); ASSERT_LT(0, usage); @@ -132,7 +133,8 @@ TEST_F(SimCacheTest, SimCache) { CheckCacheCounters(options, 1, 0, 1, 0); } ASSERT_EQ(0, simCache->GetPinnedUsage()); - ASSERT_EQ(3 * kNumBlocks + 1, simCache->get_lookup_counter()); + ASSERT_EQ(3 * kNumBlocks + 1, + simCache->get_hit_counter() + simCache->get_miss_counter()); ASSERT_EQ(6, simCache->get_hit_counter()); }