Persistent Cache: Expose stats to user via public API

Summary:
Exposing persistent cache stats (counters) to the user via public API.
Closes https://github.com/facebook/rocksdb/pull/1485

Differential Revision: D4155274

Pulled By: siying

fbshipit-source-id: 30a9f50
main
Karthikeyan Radhakrishnan 8 years ago committed by Facebook Github Bot
parent f2a8f92a15
commit 4118e13330
  1. 4
      db/db_test2.cc
  2. 8
      include/rocksdb/persistent_cache.h
  3. 64
      utilities/persistent_cache/block_cache_tier.cc
  4. 6
      utilities/persistent_cache/block_cache_tier.h
  5. 2
      utilities/persistent_cache/persistent_cache_test.cc
  6. 17
      utilities/persistent_cache/persistent_cache_tier.cc
  7. 6
      utilities/persistent_cache/persistent_cache_tier.h
  8. 27
      utilities/persistent_cache/volatile_tier_impl.cc
  9. 5
      utilities/persistent_cache/volatile_tier_impl.h

@ -1470,6 +1470,10 @@ class MockPersistentCache : public PersistentCache {
virtual ~MockPersistentCache() {}
PersistentCache::StatsType Stats() override {
return PersistentCache::StatsType();
}
Status Insert(const Slice& page_key, const char* data,
const size_t size) override {
MutexLock _(&lock_);

@ -24,6 +24,8 @@ namespace rocksdb {
// cache interface is specifically designed for persistent read cache.
class PersistentCache {
public:
typedef std::vector<std::map<std::string, double>> StatsType;
virtual ~PersistentCache() {}
// Insert to page cache
@ -46,6 +48,12 @@ class PersistentCache {
//
// True if the cache is configured to store uncompressed data else false
virtual bool IsCompressed() = 0;
// Return stats as map of {string, double} per-tier
//
// Persistent cache can be initialized as a tier of caches. The stats are per
// tire top-down
virtual StatsType Stats() = 0;
};
// Factor method to create a new persistent cache

@ -123,34 +123,42 @@ Status BlockCacheTier::Close() {
return Status::OK();
}
std::string BlockCacheTier::PrintStats() {
std::ostringstream os;
os << "persistentcache.blockcachetier.bytes_piplined: "
<< stats_.bytes_pipelined_.ToString() << std::endl
<< "persistentcache.blockcachetier.bytes_written: "
<< stats_.bytes_written_.ToString() << std::endl
<< "persistentcache.blockcachetier.bytes_read: "
<< stats_.bytes_read_.ToString() << std::endl
<< "persistentcache.blockcachetier.insert_dropped"
<< stats_.insert_dropped_ << std::endl
<< "persistentcache.blockcachetier.cache_hits: " << stats_.cache_hits_
<< std::endl
<< "persistentcache.blockcachetier.cache_misses: " << stats_.cache_misses_
<< std::endl
<< "persistentcache.blockcachetier.cache_errors: " << stats_.cache_errors_
<< std::endl
<< "persistentcache.blockcachetier.cache_hits_pct: "
<< stats_.CacheHitPct() << std::endl
<< "persistentcache.blockcachetier.cache_misses_pct: "
<< stats_.CacheMissPct() << std::endl
<< "persistentcache.blockcachetier.read_hit_latency: "
<< stats_.read_hit_latency_.ToString() << std::endl
<< "persistentcache.blockcachetier.read_miss_latency: "
<< stats_.read_miss_latency_.ToString() << std::endl
<< "persistenetcache.blockcachetier.write_latency: "
<< stats_.write_latency_.ToString() << std::endl
<< PersistentCacheTier::PrintStats();
return os.str();
template<class T>
void Add(std::map<std::string, double>* stats, const std::string& key,
const T& t) {
stats->insert({key, static_cast<const double>(t)});
}
PersistentCache::StatsType BlockCacheTier::Stats() {
std::map<std::string, double> stats;
Add(&stats, "persistentcache.blockcachetier.bytes_piplined",
stats_.bytes_pipelined_.Average());
Add(&stats, "persistentcache.blockcachetier.bytes_written",
stats_.bytes_written_.Average());
Add(&stats, "persistentcache.blockcachetier.bytes_read",
stats_.bytes_read_.Average());
Add(&stats, "persistentcache.blockcachetier.insert_dropped",
stats_.insert_dropped_);
Add(&stats, "persistentcache.blockcachetier.cache_hits",
stats_.cache_hits_);
Add(&stats, "persistentcache.blockcachetier.cache_misses",
stats_.cache_misses_);
Add(&stats, "persistentcache.blockcachetier.cache_errors",
stats_.cache_errors_);
Add(&stats, "persistentcache.blockcachetier.cache_hits_pct",
stats_.CacheHitPct());
Add(&stats, "persistentcache.blockcachetier.cache_misses_pct",
stats_.CacheMissPct());
Add(&stats, "persistentcache.blockcachetier.read_hit_latency",
stats_.read_hit_latency_.Average());
Add(&stats, "persistentcache.blockcachetier.read_miss_latency",
stats_.read_miss_latency_.Average());
Add(&stats, "persistenetcache.blockcachetier.write_latency",
stats_.write_latency_.Average());
auto out = PersistentCacheTier::Stats();
out.push_back(stats);
return out;
}
Status BlockCacheTier::Insert(const Slice& key, const char* data,

@ -64,7 +64,7 @@ class BlockCacheTier : public PersistentCacheTier {
bool IsCompressed() override { return opt_.is_compressed; }
std::string PrintStats() override;
PersistentCache::StatsType Stats() override;
void TEST_Flush() override {
while (insert_ops_.Size()) {
@ -110,7 +110,7 @@ class BlockCacheTier : public PersistentCacheTier {
Status CleanupCacheFolder(const std::string& folder);
// Statistics
struct Stats {
struct Statistics {
HistogramImpl bytes_pipelined_;
HistogramImpl bytes_written_;
HistogramImpl bytes_read_;
@ -143,7 +143,7 @@ class BlockCacheTier : public PersistentCacheTier {
ThreadedWriter writer_; // Writer threads
BlockCacheTierMetadata metadata_; // Cache meta data manager
std::atomic<uint64_t> size_{0}; // Size of the cache
Stats stats_; // Statistics
Statistics stats_; // Statistics
};
} // namespace rocksdb

@ -244,6 +244,8 @@ TEST_F(PersistentCacheTierTest, FactoryTest) {
/*size=*/1 * 1024 * 1024 * 1024, log, nvm_opt,
&cache));
ASSERT_TRUE(cache);
ASSERT_EQ(cache->Stats().size(), 1);
ASSERT_TRUE(cache->Stats()[0].size());
cache.reset();
}
}

@ -8,6 +8,7 @@
#include "utilities/persistent_cache/persistent_cache_tier.h"
#include <string>
#include <sstream>
namespace rocksdb {
@ -40,17 +41,21 @@ bool PersistentCacheTier::Erase(const Slice& key) {
}
std::string PersistentCacheTier::PrintStats() {
if (next_tier_) {
return next_tier_->PrintStats();
std::ostringstream os;
for (auto tier_stats : Stats()) {
os << "---- next tier -----" << std::endl;
for (auto stat : tier_stats) {
os << stat.first << ": " << stat.second << std::endl;
}
}
return std::string();
return os.str();
}
std::vector<PersistentCacheTier::TierStats> PersistentCacheTier::Stats() {
PersistentCache::StatsType PersistentCacheTier::Stats() {
if (next_tier_) {
return next_tier_->Stats();
}
return std::vector<TierStats>{};
return PersistentCache::StatsType{};
}
//
@ -77,7 +82,7 @@ bool PersistentTieredCache::Erase(const Slice& key) {
return tiers_.front()->Erase(key);
}
std::vector<PersistentCacheTier::TierStats> PersistentTieredCache::Stats() {
PersistentCache::StatsType PersistentTieredCache::Stats() {
assert(!tiers_.empty());
return tiers_.front()->Stats();
}

@ -231,7 +231,6 @@ struct PersistentCacheConfig {
class PersistentCacheTier : public PersistentCache {
public:
typedef std::shared_ptr<PersistentCacheTier> Tier;
typedef std::map<std::string, double> TierStats;
virtual ~PersistentCacheTier() {}
@ -250,8 +249,7 @@ class PersistentCacheTier : public PersistentCache {
// Print stats to string recursively
virtual std::string PrintStats();
// Expose stats
virtual std::vector<TierStats> Stats();
virtual PersistentCache::StatsType Stats();
// Insert to page cache
virtual Status Insert(const Slice& page_key, const char* data,
@ -296,7 +294,7 @@ class PersistentTieredCache : public PersistentCacheTier {
Status Close() override;
bool Erase(const Slice& key) override;
std::string PrintStats() override;
std::vector<TierStats> Stats() override;
PersistentCache::StatsType Stats() override;
Status Insert(const Slice& page_key, const char* data,
const size_t size) override;
Status Lookup(const Slice& page_key, std::unique_ptr<char[]>* data,

@ -18,8 +18,8 @@ void VolatileCacheTier::DeleteCacheData(VolatileCacheTier::CacheData* data) {
VolatileCacheTier::~VolatileCacheTier() { index_.Clear(&DeleteCacheData); }
std::vector<PersistentCacheTier::TierStats> VolatileCacheTier::Stats() {
PersistentCacheTier::TierStats stat;
PersistentCache::StatsType VolatileCacheTier::Stats() {
std::map<std::string, double> stat;
stat.insert({"persistent_cache.volatile_cache.hits",
static_cast<double>(stats_.cache_hits_)});
stat.insert({"persistent_cache.volatile_cache.misses",
@ -33,26 +33,9 @@ std::vector<PersistentCacheTier::TierStats> VolatileCacheTier::Stats() {
stat.insert({"persistent_cache.volatile_cache.miss_pct",
static_cast<double>(stats_.CacheMissPct())});
std::vector<PersistentCacheTier::TierStats> tier_stats;
if (next_tier()) {
tier_stats = next_tier()->Stats();
}
tier_stats.push_back(stat);
return tier_stats;
}
std::string VolatileCacheTier::PrintStats() {
std::ostringstream ss;
ss << "pagecache.volatilecache.hits: " << stats_.cache_hits_ << std::endl
<< "pagecache.volatilecache.misses: " << stats_.cache_misses_ << std::endl
<< "pagecache.volatilecache.inserts: " << stats_.cache_inserts_
<< std::endl
<< "pagecache.volatilecache.evicts: " << stats_.cache_evicts_ << std::endl
<< "pagecache.volatilecache.hit_pct: " << stats_.CacheHitPct() << std::endl
<< "pagecache.volatilecache.miss_pct: " << stats_.CacheMissPct()
<< std::endl
<< PersistentCacheTier::PrintStats();
return ss.str();
auto out = PersistentCacheTier::Stats();
out.push_back(stat);
return out;
}
Status VolatileCacheTier::Insert(const Slice& page_key, const char* data,

@ -63,10 +63,7 @@ class VolatileCacheTier : public PersistentCacheTier {
bool Erase(const Slice& key) override;
// Expose stats as map
std::vector<TierStats> Stats() override;
// Print stats to string
std::string PrintStats() override;
PersistentCache::StatsType Stats() override;
private:
//

Loading…
Cancel
Save