From c49d704656fdd9fd27f7a7e9fc2f88c6b912c20d Mon Sep 17 00:00:00 2001 From: Siying Dong Date: Tue, 18 Apr 2017 16:30:51 -0700 Subject: [PATCH] Add DB:ResetStats() Summary: Add a function to allow users to reset internal stats without restarting the DB. Closes https://github.com/facebook/rocksdb/pull/2167 Differential Revision: D4907939 Pulled By: siying fbshipit-source-id: ab2dd85b88aabe9380da7485320a1d460d3e1f68 --- HISTORY.md | 2 ++ db/db_impl.cc | 10 +++++++ db/db_impl.h | 2 ++ db/db_properties_test.cc | 7 +++++ db/internal_stats.h | 62 +++++++++++++++++++++++++++++++++++++++- include/rocksdb/db.h | 7 +++++ tools/db_bench_tool.cc | 12 ++++++++ 7 files changed, 101 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index a98e9c2aa..aa3c0d43b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ # Rocksdb Change Log ## Unreleased +### New Features +* DB::ResetStats() to reset internal stats. ## 5.4.0 (04/11/2017) ### Public API Change diff --git a/db/db_impl.cc b/db/db_impl.cc index 62d0092fc..fd934bc3f 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1666,6 +1666,16 @@ bool DBImpl::GetIntPropertyInternal(ColumnFamilyData* cfd, } } +#ifndef ROCKSDB_LITE +Status DBImpl::ResetStats() { + InstrumentedMutexLock l(&mutex_); + for (auto* cfd : *versions_->GetColumnFamilySet()) { + cfd->internal_stats()->Clear(); + } + return Status::OK(); +} +#endif // ROCKSDB_LITE + bool DBImpl::GetAggregatedIntProperty(const Slice& property, uint64_t* aggregated_value) { const DBPropertyInfo* property_info = GetPropertyInfo(property); diff --git a/db/db_impl.h b/db/db_impl.h index 2b75d53bb..c6b4d756c 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -193,6 +193,8 @@ class DBImpl : public DB { virtual SequenceNumber GetLatestSequenceNumber() const override; #ifndef ROCKSDB_LITE + using DB::ResetStats; + virtual Status ResetStats() override; virtual Status DisableFileDeletions() override; virtual Status EnableFileDeletions(bool force) override; virtual int IsFileDeletionsEnabled() const; diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc index ebc5f96e0..5ce6fb8d7 100644 --- a/db/db_properties_test.cc +++ b/db/db_properties_test.cc @@ -425,6 +425,13 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) { ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + + // Clear internal stats + dbfull()->ResetStats(); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop)); + ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); } TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) { diff --git a/db/internal_stats.h b/db/internal_stats.h index 584184437..86f376092 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -189,6 +189,20 @@ class InternalStats { num_dropped_records(c.num_dropped_records), count(c.count) {} + void Clear() { + this->micros = 0; + this->bytes_read_non_output_levels = 0; + this->bytes_read_output_level = 0; + this->bytes_written = 0; + this->bytes_moved = 0; + this->num_input_files_in_non_output_levels = 0; + this->num_input_files_in_output_level = 0; + this->num_output_files = 0; + this->num_input_records = 0; + this->num_dropped_records = 0; + this->count = 0; + } + void Add(const CompactionStats& c) { this->micros += c.micros; this->bytes_read_non_output_levels += c.bytes_read_non_output_levels; @@ -222,6 +236,26 @@ class InternalStats { } }; + void Clear() { + for (int i = 0; i < INTERNAL_DB_STATS_ENUM_MAX; i++) { + db_stats_[i].store(0); + } + for (int i = 0; i < INTERNAL_CF_STATS_ENUM_MAX; i++) { + cf_stats_count_[i] = 0; + cf_stats_value_[i] = 0; + } + for (auto& comp_stat : comp_stats_) { + comp_stat.Clear(); + } + for (auto& h : file_read_latency_) { + h.Clear(); + } + cf_stats_snapshot_.Clear(); + db_stats_snapshot_.Clear(); + bg_error_count_ = 0; + started_at_ = env_->NowMicros(); + } + void AddCompactionStats(int level, const CompactionStats& stats) { comp_stats_[level].Add(stats); } @@ -319,6 +353,20 @@ class InternalStats { ingest_files_addfile(0), ingest_l0_files_addfile(0), ingest_keys_addfile(0) {} + + void Clear() { + comp_stats.Clear(); + ingest_bytes_flush = 0; + stall_count = 0; + compact_bytes_write = 0; + compact_bytes_read = 0; + compact_micros = 0; + seconds_up = 0; + ingest_bytes_addfile = 0; + ingest_files_addfile = 0; + ingest_l0_files_addfile = 0; + ingest_keys_addfile = 0; + } } cf_stats_snapshot_; struct DBStatsSnapshot { @@ -350,6 +398,18 @@ class InternalStats { num_keys_written(0), write_stall_micros(0), seconds_up(0) {} + + void Clear() { + ingest_bytes = 0; + wal_bytes = 0; + wal_synced = 0; + write_with_wal = 0; + write_other = 0; + write_self = 0; + num_keys_written = 0; + write_stall_micros = 0; + seconds_up = 0; + } } db_stats_snapshot_; // Handler functions for getting property values. They use "value" as a value- @@ -420,7 +480,7 @@ class InternalStats { const int number_levels_; Env* env_; ColumnFamilyData* cfd_; - const uint64_t started_at_; + uint64_t started_at_; }; #else diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 723e4dea3..988103dfd 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -611,6 +611,13 @@ class DB { return GetIntProperty(DefaultColumnFamily(), property, value); } + // Reset internal stats for DB and all column families. + // Note this doesn't reset options.statistics as it is not owned by + // DB. + virtual Status ResetStats() { + return Status::NotSupported("Not implemented"); + } + // Same as GetIntProperty(), but this one returns the aggregated int // property from all column families. virtual bool GetAggregatedIntProperty(const Slice& property, diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index a52738df3..969eb0ec6 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -176,6 +176,7 @@ DEFINE_string( "Meta operations:\n" "\tcompact -- Compact the entire DB\n" "\tstats -- Print DB stats\n" + "\tresetstats -- Reset DB stats\n" "\tlevelstats -- Print the number of files and bytes per level\n" "\tsstables -- Print sstable info\n" "\theapprofile -- Dump a heap profile (if supported by this" @@ -2430,6 +2431,8 @@ void VerifyDBFromDB(std::string& truth_db_name) { method = &Benchmark::TimeSeries; } else if (name == "stats") { PrintStats("rocksdb.stats"); + } else if (name == "resetstats") { + ResetStats(); } else if (name == "verify") { VerifyDBFromDB(FLAGS_truth_db); } else if (name == "levelstats") { @@ -5012,6 +5015,15 @@ void VerifyDBFromDB(std::string& truth_db_name) { db->CompactRange(CompactRangeOptions(), nullptr, nullptr); } + void ResetStats() { + if (db_.db != nullptr) { + db_.db->ResetStats(); + } + for (const auto& db_with_cfh : multi_dbs_) { + db_with_cfh.db->ResetStats(); + } + } + void PrintStats(const char* key) { if (db_.db != nullptr) { PrintStats(db_.db, key, false);