From efc361ef7de6bdfcad31a425965ef0fec1c3e8e7 Mon Sep 17 00:00:00 2001 From: Andrew Kryczka Date: Wed, 26 Apr 2017 15:19:50 -0700 Subject: [PATCH] Add user stats Reset API Summary: It resets all the ticker and histogram stats to zero. Needed to change the locking a bit since Reset() is the only operation that manipulates multiple tickers/histograms together, and that operation should be seen as atomic by other operations that access tickers/histograms. Closes https://github.com/facebook/rocksdb/pull/2213 Differential Revision: D4952232 Pulled By: ajkr fbshipit-source-id: c0475c3e4c7b940120d53891b69c3091149a0679 --- HISTORY.md | 1 + db/db_statistics_test.cc | 25 ++++++++++++++++ include/rocksdb/statistics.h | 7 +++++ monitoring/statistics.cc | 57 ++++++++++++++++++++++++++++-------- monitoring/statistics.h | 10 +++++-- 5 files changed, 85 insertions(+), 15 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 498d9324e..7b51d37a0 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,7 @@ ## Unreleased ### New Features * DB::ResetStats() to reset internal stats. +* Statistics::Reset() to reset user stats. * ldb add option --try_load_options, which will open DB with its own option file. ## 5.4.0 (04/11/2017) diff --git a/db/db_statistics_test.cc b/db/db_statistics_test.cc index fbfeea357..235efef7d 100644 --- a/db/db_statistics_test.cc +++ b/db/db_statistics_test.cc @@ -115,6 +115,31 @@ TEST_F(DBStatisticsTest, MutexWaitStats) { ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0); } +TEST_F(DBStatisticsTest, ResetStats) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = rocksdb::CreateDBStatistics(); + DestroyAndReopen(options); + for (int i = 0; i < 2; ++i) { + // pick arbitrary ticker and histogram. On first iteration they're zero + // because db is unused. On second iteration they're zero due to Reset(). + ASSERT_EQ(0, TestGetTickerCount(options, NUMBER_KEYS_WRITTEN)); + HistogramData histogram_data; + options.statistics->histogramData(DB_WRITE, &histogram_data); + ASSERT_EQ(0.0, histogram_data.max); + + if (i == 0) { + // The Put() makes some of the ticker/histogram stats nonzero until we + // Reset(). + ASSERT_OK(Put("hello", "rocksdb")); + ASSERT_EQ(1, TestGetTickerCount(options, NUMBER_KEYS_WRITTEN)); + options.statistics->histogramData(DB_WRITE, &histogram_data); + ASSERT_GT(histogram_data.max, 0.0); + options.statistics->Reset(); + } + } +} + } // namespace rocksdb int main(int argc, char** argv) { diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index 7f343bb23..c81794332 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -13,6 +13,8 @@ #include #include +#include "rocksdb/status.h" + namespace rocksdb { /** @@ -443,6 +445,11 @@ class Statistics { virtual uint64_t getAndResetTickerCount(uint32_t tickerType) = 0; virtual void measureTime(uint32_t histogramType, uint64_t time) = 0; + // Resets all ticker and histogram stats + virtual Status Reset() { + return Status::NotSupported("Not implemented"); + } + // String representation of the statistic object. virtual std::string ToString() const { // Do nothing by default diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index c9b86cdcc..fb5634f76 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -33,6 +33,10 @@ StatisticsImpl::~StatisticsImpl() {} uint64_t StatisticsImpl::getTickerCount(uint32_t tickerType) const { MutexLock lock(&aggregate_lock_); + return getTickerCountLocked(tickerType); +} + +uint64_t StatisticsImpl::getTickerCountLocked(uint32_t tickerType) const { assert( enable_internal_stats_ ? tickerType < INTERNAL_TICKER_ENUM_MAX : @@ -68,6 +72,12 @@ StatisticsImpl::HistogramInfo::getMergedHistogram() const { void StatisticsImpl::histogramData(uint32_t histogramType, HistogramData* const data) const { + MutexLock lock(&aggregate_lock_); + histogramDataLocked(histogramType, data); +} + +void StatisticsImpl::histogramDataLocked(uint32_t histogramType, + HistogramData* const data) const { assert( enable_internal_stats_ ? histogramType < INTERNAL_HISTOGRAM_ENUM_MAX : @@ -76,6 +86,7 @@ void StatisticsImpl::histogramData(uint32_t histogramType, } std::string StatisticsImpl::getHistogramString(uint32_t histogramType) const { + MutexLock lock(&aggregate_lock_); assert(enable_internal_stats_ ? histogramType < INTERNAL_HISTOGRAM_ENUM_MAX : histogramType < HISTOGRAM_ENUM_MAX); return histograms_[histogramType].getMergedHistogram()->ToString(); @@ -108,23 +119,27 @@ StatisticsImpl::ThreadHistogramInfo* StatisticsImpl::getThreadHistogramInfo( void StatisticsImpl::setTickerCount(uint32_t tickerType, uint64_t count) { { MutexLock lock(&aggregate_lock_); - assert(enable_internal_stats_ ? tickerType < INTERNAL_TICKER_ENUM_MAX - : tickerType < TICKER_ENUM_MAX); - if (tickerType < TICKER_ENUM_MAX || enable_internal_stats_) { - tickers_[tickerType].thread_value->Fold( - [](void* curr_ptr, void* res) { - static_cast*>(curr_ptr)->store( - 0, std::memory_order_relaxed); - }, - nullptr /* res */); - tickers_[tickerType].merged_sum.store(count, std::memory_order_relaxed); - } + setTickerCountLocked(tickerType, count); } if (stats_ && tickerType < TICKER_ENUM_MAX) { stats_->setTickerCount(tickerType, count); } } +void StatisticsImpl::setTickerCountLocked(uint32_t tickerType, uint64_t count) { + assert(enable_internal_stats_ ? tickerType < INTERNAL_TICKER_ENUM_MAX + : tickerType < TICKER_ENUM_MAX); + if (tickerType < TICKER_ENUM_MAX || enable_internal_stats_) { + tickers_[tickerType].thread_value->Fold( + [](void* curr_ptr, void* res) { + static_cast*>(curr_ptr)->store( + 0, std::memory_order_relaxed); + }, + nullptr /* res */); + tickers_[tickerType].merged_sum.store(count, std::memory_order_relaxed); + } +} + uint64_t StatisticsImpl::getAndResetTickerCount(uint32_t tickerType) { uint64_t sum = 0; { @@ -176,6 +191,21 @@ void StatisticsImpl::measureTime(uint32_t histogramType, uint64_t value) { } } +Status StatisticsImpl::Reset() { + MutexLock lock(&aggregate_lock_); + for (uint32_t i = 0; i < TICKER_ENUM_MAX; ++i) { + setTickerCountLocked(i, 0); + } + for (uint32_t i = 0; i < HISTOGRAM_ENUM_MAX; ++i) { + histograms_[i].thread_value->Fold( + [](void* curr_ptr, void* res) { + static_cast(curr_ptr)->Clear(); + }, + nullptr /* res */); + } + return Status::OK(); +} + namespace { // a buffer size used for temp string buffers @@ -184,13 +214,14 @@ const int kTmpStrBufferSize = 200; } // namespace std::string StatisticsImpl::ToString() const { + MutexLock lock(&aggregate_lock_); std::string res; res.reserve(20000); for (const auto& t : TickersNameMap) { if (t.first < TICKER_ENUM_MAX || enable_internal_stats_) { char buffer[kTmpStrBufferSize]; snprintf(buffer, kTmpStrBufferSize, "%s COUNT : %" PRIu64 "\n", - t.second.c_str(), getTickerCount(t.first)); + t.second.c_str(), getTickerCountLocked(t.first)); res.append(buffer); } } @@ -198,7 +229,7 @@ std::string StatisticsImpl::ToString() const { if (h.first < HISTOGRAM_ENUM_MAX || enable_internal_stats_) { char buffer[kTmpStrBufferSize]; HistogramData hData; - histogramData(h.first, &hData); + histogramDataLocked(h.first, &hData); snprintf( buffer, kTmpStrBufferSize, "%s statistics Percentiles :=> 50 : %f 95 : %f 99 : %f 100 : %f\n", diff --git a/monitoring/statistics.h b/monitoring/statistics.h index b64dc4fb4..32b7036ca 100644 --- a/monitoring/statistics.h +++ b/monitoring/statistics.h @@ -45,6 +45,7 @@ class StatisticsImpl : public Statistics { virtual void recordTick(uint32_t ticker_type, uint64_t count) override; virtual void measureTime(uint32_t histogram_type, uint64_t value) override; + virtual Status Reset() override; virtual std::string ToString() const override; virtual bool HistEnabledForType(uint32_t type) const override; @@ -52,8 +53,8 @@ class StatisticsImpl : public Statistics { std::shared_ptr stats_shared_; Statistics* stats_; bool enable_internal_stats_; - // Synchronizes setTickerCount()/getTickerCount() operations so partially - // completed setTickerCount() won't be visible. + // Synchronizes anything that operates on other threads' thread-specific data + // such that operations like Reset() can be performed atomically. mutable port::Mutex aggregate_lock_; // Holds data maintained by each thread for implementing tickers. @@ -126,6 +127,11 @@ class StatisticsImpl : public Statistics { std::unique_ptr getMergedHistogram() const; }; + uint64_t getTickerCountLocked(uint32_t ticker_type) const; + void histogramDataLocked(uint32_t histogram_type, + HistogramData* const data) const; + void setTickerCountLocked(uint32_t ticker_type, uint64_t count); + // Returns the info for this tickerType/thread. It sets a new info with zeroed // counter if none exists. ThreadTickerInfo* getThreadTickerInfo(uint32_t ticker_type);