NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

File level histogram should be printed per CF, not per DB

Browse Source 
Summary:
Currently level histogram is only printed out for DB stats and for default CF. This is confusing. Change to print for every CF instead.
Closes https://github.com/facebook/rocksdb/pull/2126

Differential Revision: D4865373

Pulled By: siying

fbshipit-source-id: 1c853e0ac66e00120ee931cabc9daf69ccc2d577
main
Siying Dong 7 years ago committed by Facebook Github Bot
parent 9300ef5455
commit 8f47a97512
5 changed files with 100 additions and 28 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 10
      db/db_impl.cc
  2. 40
      db/db_properties_test.cc
  3. 55
      db/internal_stats.cc
  4. 4
      db/internal_stats.h
  5. 19
      include/rocksdb/db.h

10
db/db_impl.cc
Unescape View File

@ -432,12 +432,16 @@ void DBImpl::MaybeDumpStats() {
std::string stats;
{
InstrumentedMutexLock l(&mutex_);
default_cf_internal_stats_->GetStringProperty(
*db_property_info, DB::Properties::kDBStats, &stats);
for (auto cfd : *versions_->GetColumnFamilySet()) {
cfd->internal_stats()->GetStringProperty(
*cf_property_info, DB::Properties::kCFStats, &stats);
*cf_property_info, DB::Properties::kCFStatsNoFileHistogram, &stats);
}
for (auto cfd : *versions_->GetColumnFamilySet()) {
cfd->internal_stats()->GetStringProperty(
*cf_property_info, DB::Properties::kCFFileHistogram, &stats);
}
default_cf_internal_stats_->GetStringProperty(
*db_property_info, DB::Properties::kDBStats, &stats);
}
ROCKS_LOG_WARN(immutable_db_options_.info_log,
"------- DUMPING STATS -------");

40
db/db_properties_test.cc
Unescape View File

@ -342,7 +342,7 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
BlockBasedTableOptions table_options;
table_options.no_block_cache = true;
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
int key_index = 0;
Random rnd(301);
for (int num = 0; num < 8; num++) {
@ -359,25 +359,26 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
for (int key = 0; key < key_index; key++) {
Get(Key(key));
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// Reopen and issue Get(). See thee latency tracked
Reopen(options);
ReopenWithColumnFamilies({"default", "pikachu"}, options);
dbfull()->TEST_WaitForCompact();
for (int key = 0; key < key_index; key++) {
Get(Key(key));
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
"rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// Reopen and issue iterating. See thee latency tracked
Reopen(options);
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
@ -386,22 +387,41 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
}
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// CF 1 should show no histogram.
ASSERT_TRUE(
dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// put something and read it back , CF 1 should show histogram.
Put(1, "foo", "bar");
Flush(1);
dbfull()->TEST_WaitForCompact();
ASSERT_EQ("bar", Get(1, "foo"));
ASSERT_TRUE(
dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// options.max_open_files preloads table readers.
options.max_open_files = -1;
Reopen(options);
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
"rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
for (int key = 0; key < key_index; key++) {
Get(Key(key));
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));

55
db/internal_stats.cc
Unescape View File

@ -195,6 +195,9 @@ static const std::string compression_ratio_at_level_prefix =
static const std::string allstats = "stats";
static const std::string sstables = "sstables";
static const std::string cfstats = "cfstats";
static const std::string cfstats_no_file_histogram =
"cfstats-no-file-histogram";
static const std::string cf_file_histogram = "cf-file-histogram";
static const std::string dbstats = "dbstats";
static const std::string levelstats = "levelstats";
static const std::string num_immutable_mem_table = "num-immutable-mem-table";
@ -248,6 +251,10 @@ const std::string DB::Properties::kCompressionRatioAtLevelPrefix =
const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
const std::string DB::Properties::kCFStatsNoFileHistogram =
rocksdb_prefix + cfstats_no_file_histogram;
const std::string DB::Properties::kCFFileHistogram =
rocksdb_prefix + cf_file_histogram;
const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
const std::string DB::Properties::kNumImmutableMemTable =
@ -324,6 +331,11 @@ const std::unordered_map<std::string, DBPropertyInfo>
{DB::Properties::kCFStats,
{false, &InternalStats::HandleCFStats, nullptr,
&InternalStats::HandleCFMapStats}},
{DB::Properties::kCFStatsNoFileHistogram,
{false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr,
nullptr}},
{DB::Properties::kCFFileHistogram,
{false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr}},
{DB::Properties::kDBStats,
{false, &InternalStats::HandleDBStats, nullptr, nullptr}},
{DB::Properties::kSSTables,
@ -512,6 +524,17 @@ bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
return true;
}
bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
Slice suffix) {
DumpCFStatsNoFileHistogram(value);
return true;
}
bool InternalStats::HandleCFFileHistogram(std::string* value, Slice suffix) {
DumpCFFileHistogram(value);
return true;
}
bool InternalStats::HandleDBStats(std::string* value, Slice suffix) {
DumpDBStats(value);
return true;
@ -855,16 +878,6 @@ void InternalStats::DumpDBStats(std::string* value) {
10000.0 / std::max(interval_seconds_up, 0.001));
value->append(buf);
for (int level = 0; level < number_levels_; level++) {
if (!file_read_latency_[level].Empty()) {
char buf2[5000];
snprintf(buf2, sizeof(buf2),
"** Level %d read latency histogram (micros):\n%s\n", level,
file_read_latency_[level].ToString().c_str());
value->append(buf2);
}
}
db_stats_snapshot_.seconds_up = seconds_up;
db_stats_snapshot_.ingest_bytes = user_bytes_written;
db_stats_snapshot_.write_other = write_other;
@ -968,6 +981,11 @@ void InternalStats::DumpCFMapStats(
}
void InternalStats::DumpCFStats(std::string* value) {
DumpCFStatsNoFileHistogram(value);
DumpCFFileHistogram(value);
}
void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
char buf[2000];
// Per-ColumnFamily stats
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
@ -1126,6 +1144,23 @@ void InternalStats::DumpCFStats(std::string* value) {
cf_stats_snapshot_.stall_count = total_stall_count;
}
void InternalStats::DumpCFFileHistogram(std::string* value) {
char buf[2000];
snprintf(buf, sizeof(buf),
"\n** File Read Latency Histogram By Level [%s] **\n",
cfd_->GetName().c_str());
value->append(buf);
for (int level = 0; level < number_levels_; level++) {
if (!file_read_latency_[level].Empty()) {
char buf2[5000];
snprintf(buf2, sizeof(buf2),
"** Level %d read latency histogram (micros):\n%s\n", level,
file_read_latency_[level].ToString().c_str());
value->append(buf2);
}
}
}
#else

4
db/internal_stats.h
Unescape View File

@ -277,6 +277,8 @@ class InternalStats {
std::map<int, std::map<LevelStatType, double>>* level_stats,
CompactionStats* compaction_stats_sum);
void DumpCFStats(std::string* value);
void DumpCFStatsNoFileHistogram(std::string* value);
void DumpCFFileHistogram(std::string* value);
// Per-DB stats
std::atomic<uint64_t> db_stats_[INTERNAL_DB_STATS_ENUM_MAX];
@ -358,6 +360,8 @@ class InternalStats {
bool HandleStats(std::string* value, Slice suffix);
bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
bool HandleCFStats(std::string* value, Slice suffix);
bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix);
bool HandleCFFileHistogram(std::string* value, Slice suffix);
bool HandleDBStats(std::string* value, Slice suffix);
bool HandleSsTables(std::string* value, Slice suffix);
bool HandleAggregatedTableProperties(std::string* value, Slice suffix);

19
include/rocksdb/db.h
Unescape View File

@ -400,15 +400,24 @@ class DB {
// SST files.
static const std::string kSSTables;
// "rocksdb.cfstats" - returns a multi-line string with general column
// family stats per-level over db's lifetime ("L<n>"), aggregated over
// db's lifetime ("Sum"), and aggregated over the interval since the
// last retrieval ("Int").
// "rocksdb.cfstats" - Both of "rocksdb.cfstats-no-file-histogram" and
// "rocksdb.cf-file-histogram" together. See below for description
// of the two.
static const std::string kCFStats;
// "rocksdb.cfstats-no-file-histogram" - returns a multi-line string with
// general columm family stats per-level over db's lifetime ("L<n>"),
// aggregated over db's lifetime ("Sum"), and aggregated over the
// interval since the last retrieval ("Int").
// It could also be used to return the stats in the format of the map.
// In this case there will a pair of string to array of double for
// each level as well as for "Sum". "Int" stats will not be affected
// when this form of stats are retrived.
static const std::string kCFStats;
static const std::string kCFStatsNoFileHistogram;
// "rocksdb.cf-file-histogram" - print out how many file reads to every
// level, as well as the histogram of latency of single requests.
static const std::string kCFFileHistogram;
// "rocksdb.dbstats" - returns a multi-line string with general database
// stats, both cumulative (over the db's lifetime) and interval (since

Write Preview
Loading…
Cancel
Save