NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Block cache analyzer: Add more stats (#5516)

Browse Source 
Summary:
This PR provides more command line options for block cache analyzer to better understand block cache access pattern.
-analyze_bottom_k_access_count_blocks
-analyze_top_k_access_count_blocks
-reuse_lifetime_labels
-reuse_lifetime_buckets
-analyze_callers
-access_count_buckets
-analyze_blocks_reuse_k_reuse_window
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5516

Test Plan: make clean && COMPILE_WITH_ASAN=1 make check -j32

Differential Revision: D16037440

Pulled By: HaoyuHuang

fbshipit-source-id: b9a4ac0d4712053fab910732077a4d4b91400bc8
main
haoyuhuang 5 years ago committed by Facebook Github Bot
parent 1a59b6e2a9
commit 3e9c5a3523
5 changed files with 1312 additions and 328 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1239
      tools/block_cache_trace_analyzer.cc
  2. 134
      tools/block_cache_trace_analyzer.h
  3. 262
      tools/block_cache_trace_analyzer_test.cc
  4. 2
      trace_replay/block_cache_tracer.cc
  5. 3
      trace_replay/block_cache_tracer.h

1239
tools/block_cache_trace_analyzer.cc
View File

File diff suppressed because it is too large Load Diff

134
tools/block_cache_trace_analyzer.h
Unescape View File

@ -9,13 +9,13 @@
#include <set>
#include <vector>
#include "db/dbformat.h"
#include "rocksdb/env.h"
#include "rocksdb/utilities/sim_cache.h"
#include "trace_replay/block_cache_tracer.h"
#include "utilities/simulator_cache/cache_simulator.h"
namespace rocksdb {
// Statistics of a block.
struct BlockAccessInfo {
uint64_t num_accesses = 0;
@ -23,11 +23,12 @@ struct BlockAccessInfo {
uint64_t first_access_time = 0;
uint64_t last_access_time = 0;
uint64_t num_keys = 0;
std::map<std::string, uint64_t>
std::map<std::string, std::map<TableReaderCaller, uint64_t>>
key_num_access_map; // for keys exist in this block.
std::map<std::string, uint64_t>
std::map<std::string, std::map<TableReaderCaller, uint64_t>>
non_exist_key_num_access_map; // for keys do not exist in this block.
uint64_t num_referenced_key_exist_in_block = 0;
uint64_t referenced_data_size = 0;
std::map<TableReaderCaller, uint64_t> caller_num_access_map;
// caller:timestamp:number_of_accesses. The granularity of the timestamp is
// seconds.
@ -39,6 +40,12 @@ struct BlockAccessInfo {
std::map<uint64_t, uint64_t> reuse_distance_count;
void AddAccess(const BlockCacheTraceRecord& access) {
if (block_size != 0 && access.block_size != 0) {
assert(block_size == access.block_size);
}
if (num_keys != 0 && access.num_keys_in_block != 0) {
assert(num_keys == access.num_keys_in_block);
}
if (first_access_time == 0) {
first_access_time = access.access_timestamp;
}
@ -54,10 +61,18 @@ struct BlockAccessInfo {
access.caller)) {
num_keys = access.num_keys_in_block;
if (access.referenced_key_exist_in_block == Boolean::kTrue) {
key_num_access_map[access.referenced_key]++;
if (key_num_access_map.find(access.referenced_key) ==
key_num_access_map.end()) {
referenced_data_size += access.referenced_data_size;
}
key_num_access_map[access.referenced_key][access.caller]++;
num_referenced_key_exist_in_block++;
if (referenced_data_size > block_size && block_size != 0) {
ParsedInternalKey internal_key;
ParseInternalKey(access.referenced_key, &internal_key);
}
} else {
non_exist_key_num_access_map[access.referenced_key]++;
non_exist_key_num_access_map[access.referenced_key][access.caller]++;
}
}
}
@ -83,6 +98,7 @@ class BlockCacheTraceAnalyzer {
public:
BlockCacheTraceAnalyzer(
const std::string& trace_file_path, const std::string& output_dir,
bool compute_reuse_distance,
std::unique_ptr<BlockCacheTraceSimulator>&& cache_simulator);
~BlockCacheTraceAnalyzer() = default;
// No copy and move.
@ -122,7 +138,8 @@ class BlockCacheTraceAnalyzer {
// Print access count distribution and the distribution break down by block
// type and column family.
void PrintAccessCountStats() const;
void PrintAccessCountStats(bool user_access_only, uint32_t bottom_k,
uint32_t top_k) const;
// Print data block accesses by user Get and Multi-Get.
// It prints out 1) A histogram on the percentage of keys accessed in a data
@ -131,24 +148,93 @@ class BlockCacheTraceAnalyzer {
// accesses on keys exist in a data block and its break down by column family.
void PrintDataBlockAccessStats() const;
// Write the percentage of accesses break down by column family into a csv
// file saved in 'output_dir'.
//
// The file is named "percentage_of_accesses_summary". The file format is
// caller,cf_0,cf_1,...,cf_n where the cf_i is the column family name found in
// the trace.
void WritePercentAccessSummaryStats() const;
// Write the percentage of accesses for the given caller break down by column
// family, level, and block type into a csv file saved in 'output_dir'.
//
// It generates two files: 1) caller_level_percentage_of_accesses_summary and
// 2) caller_bt_percentage_of_accesses_summary which break down by the level
// and block type, respectively. The file format is
// level/bt,cf_0,cf_1,...,cf_n where cf_i is the column family name found in
// the trace.
void WriteDetailedPercentAccessSummaryStats(TableReaderCaller caller) const;
// Write the access count summary into a csv file saved in 'output_dir'.
// It groups blocks by their access count.
//
// It generates two files: 1) cf_access_count_summary and 2)
// bt_access_count_summary which break down the access count by column family
// and block type, respectively. The file format is
// cf/bt,bucket_0,bucket_1,...,bucket_N.
void WriteAccessCountSummaryStats(
const std::vector<uint64_t>& access_count_buckets,
bool user_access_only) const;
// Write miss ratio curves of simulated cache configurations into a csv file
// saved in 'output_dir'.
// named "mrc" saved in 'output_dir'.
//
// The file format is
// "cache_name,num_shard_bits,capacity,miss_ratio,total_accesses".
void WriteMissRatioCurves() const;
// Write the access timeline into a csv file saved in 'output_dir'.
void WriteAccessTimeline(const std::string& label) const;
//
// The file is named "label_access_timeline".The file format is
// "time,label_1_access_per_second,label_2_access_per_second,...,label_N_access_per_second"
// where N is the number of unique labels found in the trace.
void WriteAccessTimeline(const std::string& label, uint64_t time_unit,
bool user_access_only) const;
// Write the reuse distance into a csv file saved in 'output_dir'. Reuse
// distance is defined as the cumulated size of unique blocks read between two
// consective accesses on the same block.
//
// The file is named "label_reuse_distance". The file format is
// bucket,label_1,label_2,...,label_N.
void WriteReuseDistance(const std::string& label_str,
const std::set<uint64_t>& distance_buckets) const;
const std::vector<uint64_t>& distance_buckets) const;
// Write the reuse interval into a csv file saved in 'output_dir'. Reuse
// interval is defined as the time between two consecutive accesses on the
// same block..
// same block.
//
// The file is named "label_reuse_interval". The file format is
// bucket,label_1,label_2,...,label_N.
void WriteReuseInterval(const std::string& label_str,
const std::set<uint64_t>& time_buckets) const;
const std::vector<uint64_t>& time_buckets) const;
// Write the reuse lifetime into a csv file saved in 'output_dir'. Reuse
// lifetime is defined as the time interval between the first access of a
// block and its last access.
//
// The file is named "label_reuse_lifetime". The file format is
// bucket,label_1,label_2,...,label_N.
void WriteReuseLifetime(const std::string& label_str,
const std::vector<uint64_t>& time_buckets) const;
// Write the reuse timeline into a csv file saved in 'output_dir'.
//
// The file is named
// "block_type_user_access_only_reuse_window_reuse_timeline". The file format
// is start_time,0,1,...,N where N equals trace_duration / reuse_window.
void WriteBlockReuseTimeline(uint64_t reuse_window, bool user_access_only,
TraceType block_type) const;
// Write the Get spatical locality into csv files saved in 'output_dir'.
//
// It generates three csv files. label_percent_ref_keys,
// label_percent_accesses_on_ref_keys, and
// label_percent_data_size_on_ref_keys.
void WriteGetSpatialLocality(
const std::string& label_str,
const std::vector<uint64_t>& percent_buckets) const;
const std::map<std::string, ColumnFamilyAccessInfoAggregate>&
TEST_cf_aggregates_map() const {
@ -161,28 +247,48 @@ class BlockCacheTraceAnalyzer {
std::string BuildLabel(const std::set<std::string>& labels,
const std::string& cf_name, uint64_t fd,
uint32_t level, TraceType type,
TableReaderCaller caller,
const std::string& block_key) const;
TableReaderCaller caller, uint64_t block_key) const;
void ComputeReuseDistance(BlockAccessInfo* info) const;
void RecordAccess(const BlockCacheTraceRecord& access);
void UpdateReuseIntervalStats(
const std::string& label, const std::set<uint64_t>& time_buckets,
const std::string& label, const std::vector<uint64_t>& time_buckets,
const std::map<uint64_t, uint64_t> timeline,
std::map<std::string, std::map<uint64_t, uint64_t>>*
label_time_num_reuses,
uint64_t* total_num_reuses) const;
std::string OutputPercentAccessStats(
uint64_t total_accesses,
const std::map<std::string, uint64_t>& cf_access_count) const;
void WriteStatsToFile(
const std::string& label_str, const std::vector<uint64_t>& time_buckets,
const std::string& filename_suffix,
const std::map<std::string, std::map<uint64_t, uint64_t>>& label_data,
uint64_t ntotal) const;
void TraverseBlocks(
std::function<void(const std::string& /*cf_name*/, uint64_t /*fd*/,
uint32_t /*level*/, TraceType /*block_type*/,
const std::string& /*block_key*/,
uint64_t /*block_key_id*/,
const BlockAccessInfo& /*block_access_info*/)>
block_callback) const;
rocksdb::Env* env_;
const std::string trace_file_path_;
const std::string output_dir_;
const bool compute_reuse_distance_;
BlockCacheTraceHeader header_;
std::unique_ptr<BlockCacheTraceSimulator> cache_simulator_;
std::map<std::string, ColumnFamilyAccessInfoAggregate> cf_aggregates_map_;
std::map<std::string, BlockAccessInfo*> block_info_map_;
uint64_t trace_start_timestamp_in_seconds_ = 0;
uint64_t trace_end_timestamp_in_seconds_ = 0;
};
int block_cache_trace_analyzer_tool(int argc, char** argv);

262
tools/block_cache_trace_analyzer_test.cc
Unescape View File

@ -56,6 +56,12 @@ class BlockCacheTracerTest : public testing::Test {
reuse_distance_buckets_ = "1,1K,1M,1G";
reuse_interval_labels_ = "block,all,cf,sst,level,bt,cf_sst,cf_level,cf_bt";
reuse_interval_buckets_ = "1,10,100,1000";
reuse_lifetime_labels_ = "block,all,cf,sst,level,bt,cf_sst,cf_level,cf_bt";
reuse_lifetime_buckets_ = "1,10,100,1000";
analyzing_callers_ = "Get,Iterator";
access_count_buckets_ = "2,3,4,5,10";
analyze_get_spatial_locality_labels_ = "all";
analyze_get_spatial_locality_buckets_ = "10,20,30,40,50,60,70,80,90,100";
}
~BlockCacheTracerTest() override {
@ -158,12 +164,22 @@ class BlockCacheTracerTest : public testing::Test {
"-print_access_count_stats",
"-print_data_block_access_count_stats",
"-cache_sim_warmup_seconds=0",
"-analyze_bottom_k_access_count_blocks=5",
"-analyze_top_k_access_count_blocks=5",
"-analyze_blocks_reuse_k_reuse_window=5",
"-timeline_labels=" + timeline_labels_,
"-reuse_distance_labels=" + reuse_distance_labels_,
"-reuse_distance_buckets=" + reuse_distance_buckets_,
"-reuse_interval_labels=" + reuse_interval_labels_,
"-reuse_interval_buckets=" + reuse_interval_buckets_,
};
"-reuse_lifetime_labels=" + reuse_lifetime_labels_,
"-reuse_lifetime_buckets=" + reuse_lifetime_buckets_,
"-analyze_callers=" + analyzing_callers_,
"-access_count_buckets=" + access_count_buckets_,
"-analyze_get_spatial_locality_labels=" +
analyze_get_spatial_locality_labels_,
"-analyze_get_spatial_locality_buckets=" +
analyze_get_spatial_locality_buckets_};
char arg_buffer[kArgBufferSize];
char* argv[kMaxArgCount];
int argc = 0;
@ -189,6 +205,12 @@ class BlockCacheTracerTest : public testing::Test {
std::string reuse_distance_buckets_;
std::string reuse_interval_labels_;
std::string reuse_interval_buckets_;
std::string reuse_lifetime_labels_;
std::string reuse_lifetime_buckets_;
std::string analyzing_callers_;
std::string access_count_buckets_;
std::string analyze_get_spatial_locality_labels_;
std::string analyze_get_spatial_locality_buckets_;
};
TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) {
@ -247,51 +269,65 @@ TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) {
}
{
// Validate the timeline csv files.
const uint32_t expected_num_lines = 50;
std::stringstream ss(timeline_labels_);
while (ss.good()) {
std::string l;
ASSERT_TRUE(getline(ss, l, ','));
const std::string timeline_file =
test_path_ + "/" + l + "_access_timeline";
std::ifstream infile(timeline_file);
std::string line;
uint32_t nlines = 0;
ASSERT_TRUE(getline(infile, line));
uint64_t expected_time = 1;
while (getline(infile, line)) {
std::stringstream ss_naccess(line);
uint32_t naccesses = 0;
std::string substr;
uint32_t time = 0;
while (ss_naccess.good()) {
ASSERT_TRUE(getline(ss_naccess, substr, ','));
if (time == 0) {
time = ParseUint32(substr);
continue;
const std::vector<std::string> time_units{"_60", "_3600"};
const std::vector<std::string> user_access_only_flags{"user_access_only_",
"all_access_"};
for (auto const& user_access_only : user_access_only_flags) {
for (auto const& unit : time_units) {
std::stringstream ss(timeline_labels_);
while (ss.good()) {
std::string l;
ASSERT_TRUE(getline(ss, l, ','));
if (l.find("block") == std::string::npos) {
if (unit != "_60" || user_access_only != "all_access_") {
continue;
}
}
naccesses += ParseUint32(substr);
const std::string timeline_file = test_path_ + "/" +
user_access_only + l + unit +
"_access_timeline";
std::ifstream infile(timeline_file);
std::string line;
const uint64_t expected_naccesses = 50;
const uint64_t expected_user_accesses = 30;
ASSERT_TRUE(getline(infile, line)) << timeline_file;
uint32_t naccesses = 0;
while (getline(infile, line)) {
std::stringstream ss_naccess(line);
std::string substr;
bool read_label = false;
while (ss_naccess.good()) {
ASSERT_TRUE(getline(ss_naccess, substr, ','));
if (!read_label) {
read_label = true;
continue;
}
naccesses += ParseUint32(substr);
}
}
if (user_access_only == "user_access_only_") {
ASSERT_EQ(expected_user_accesses, naccesses) << timeline_file;
} else {
ASSERT_EQ(expected_naccesses, naccesses) << timeline_file;
}
ASSERT_OK(env_->DeleteFile(timeline_file));
}
nlines++;
ASSERT_EQ(1, naccesses);
ASSERT_EQ(expected_time, time);
expected_time += 1;
}
ASSERT_EQ(expected_num_lines, nlines);
ASSERT_OK(env_->DeleteFile(timeline_file));
}
}
{
// Validate the reuse_interval and reuse_distance csv files.
std::map<std::string, std::string> test_reuse_csv_files;
test_reuse_csv_files["_reuse_interval"] = reuse_interval_labels_;
test_reuse_csv_files["_access_reuse_interval"] = reuse_interval_labels_;
test_reuse_csv_files["_reuse_distance"] = reuse_distance_labels_;
test_reuse_csv_files["_reuse_lifetime"] = reuse_lifetime_labels_;
test_reuse_csv_files["_avg_reuse_interval"] = reuse_interval_labels_;
test_reuse_csv_files["_avg_reuse_interval_naccesses"] =
reuse_interval_labels_;
for (auto const& test : test_reuse_csv_files) {
const std::string& file_suffix = test.first;
const std::string& labels = test.second;
const uint32_t expected_num_rows = 10;
const uint32_t expected_num_rows_absolute_values = 5;
const uint32_t expected_reused_blocks = 0;
const uint32_t expected_num_rows = 5;
std::stringstream ss(labels);
while (ss.good()) {
std::string l;
@ -300,7 +336,6 @@ TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) {
std::ifstream infile(reuse_csv_file);
std::string line;
ASSERT_TRUE(getline(infile, line));
uint32_t nblocks = 0;
double npercentage = 0;
uint32_t nrows = 0;
while (getline(infile, line)) {
@ -314,20 +349,162 @@ TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) {
label_read = true;
continue;
}
if (nrows < expected_num_rows_absolute_values) {
nblocks += ParseUint32(substr);
} else {
npercentage += ParseDouble(substr);
}
npercentage += ParseDouble(substr);
}
}
ASSERT_EQ(expected_num_rows, nrows);
ASSERT_EQ(expected_reused_blocks, nblocks);
ASSERT_LT(npercentage, 0);
if ("_reuse_lifetime" == test.first ||
"_avg_reuse_interval" == test.first ||
"_avg_reuse_interval_naccesses" == test.first) {
ASSERT_EQ(100, npercentage) << reuse_csv_file;
} else {
ASSERT_LT(npercentage, 0);
}
ASSERT_OK(env_->DeleteFile(reuse_csv_file));
}
}
}
{
// Validate the percentage of accesses summary.
const std::string percent_access_summary_file =
test_path_ + "/percentage_of_accesses_summary";
std::ifstream infile(percent_access_summary_file);
std::string line;
ASSERT_TRUE(getline(infile, line));
std::set<std::string> callers;
std::set<std::string> expected_callers{"Get", "MultiGet", "Iterator",
"Prefetch", "Compaction"};
while (getline(infile, line)) {
std::stringstream caller_percent(line);
std::string caller;
ASSERT_TRUE(getline(caller_percent, caller, ','));
std::string percent;
ASSERT_TRUE(getline(caller_percent, percent, ','));
ASSERT_FALSE(caller_percent.good());
callers.insert(caller);
ASSERT_EQ(20, ParseDouble(percent));
}
ASSERT_EQ(expected_callers.size(), callers.size());
for (auto caller : callers) {
ASSERT_TRUE(expected_callers.find(caller) != expected_callers.end());
}
ASSERT_OK(env_->DeleteFile(percent_access_summary_file));
}
{
// Validate the percentage of accesses summary by analyzing callers.
std::stringstream analyzing_callers(analyzing_callers_);
while (analyzing_callers.good()) {
std::string caller;
ASSERT_TRUE(getline(analyzing_callers, caller, ','));
std::vector<std::string> breakdowns{"level", "bt"};
for (auto breakdown : breakdowns) {
const std::string file_name = test_path_ + "/" + caller + "_" +
breakdown +
"_percentage_of_accesses_summary";
std::ifstream infile(file_name);
std::string line;
ASSERT_TRUE(getline(infile, line));
double sum = 0;
while (getline(infile, line)) {
std::stringstream label_percent(line);
std::string label;
ASSERT_TRUE(getline(label_percent, label, ','));
std::string percent;
ASSERT_TRUE(getline(label_percent, percent, ','));
ASSERT_FALSE(label_percent.good());
sum += ParseDouble(percent);
}
ASSERT_EQ(100, sum);
ASSERT_OK(env_->DeleteFile(file_name));
}
}
}
const std::vector<std::string> access_types{"user_access_only", "all_access"};
const std::vector<std::string> prefix{"bt", "cf"};
for (auto const& pre : prefix) {
for (auto const& access_type : access_types) {
{
// Validate the access count summary.
const std::string bt_access_count_summary = test_path_ + "/" + pre +
"_" + access_type +
"_access_count_summary";
std::ifstream infile(bt_access_count_summary);
std::string line;
ASSERT_TRUE(getline(infile, line));
double sum_percent = 0;
while (getline(infile, line)) {
std::stringstream bt_percent(line);
std::string bt;
ASSERT_TRUE(getline(bt_percent, bt, ','));
std::string percent;
ASSERT_TRUE(getline(bt_percent, percent, ','));
sum_percent += ParseDouble(percent);
}
ASSERT_EQ(100.0, sum_percent);
ASSERT_OK(env_->DeleteFile(bt_access_count_summary));
}
}
}
for (auto const& access_type : access_types) {
std::vector<std::string> block_types{"Index", "Data", "Filter"};
for (auto block_type : block_types) {
// Validate reuse block timeline.
const std::string reuse_blocks_timeline = test_path_ + "/" + block_type +
"_" + access_type +
"_5_reuse_blocks_timeline";
std::ifstream infile(reuse_blocks_timeline);
std::string line;
ASSERT_TRUE(getline(infile, line)) << reuse_blocks_timeline;
uint32_t index = 0;
while (getline(infile, line)) {
std::stringstream timeline(line);
bool start_time = false;
double sum = 0;
while (timeline.good()) {
std::string value;
ASSERT_TRUE(getline(timeline, value, ','));
if (!start_time) {
start_time = true;
continue;
}
sum += ParseDouble(value);
}
index++;
ASSERT_LT(sum, 100.0 * index + 1) << reuse_blocks_timeline;
}
ASSERT_OK(env_->DeleteFile(reuse_blocks_timeline));
}
}
std::stringstream ss(analyze_get_spatial_locality_labels_);
while (ss.good()) {
std::string l;
ASSERT_TRUE(getline(ss, l, ','));
const std::vector<std::string> spatial_locality_files{
"_percent_ref_keys", "_percent_accesses_on_ref_keys",
"_percent_data_size_on_ref_keys"};
for (auto const& spatial_locality_file : spatial_locality_files) {
const std::string filename = test_path_ + "/" + l + spatial_locality_file;
std::ifstream infile(filename);
std::string line;
ASSERT_TRUE(getline(infile, line));
double sum_percent = 0;
uint32_t nrows = 0;
while (getline(infile, line)) {
std::stringstream bt_percent(line);
std::string bt;
ASSERT_TRUE(getline(bt_percent, bt, ','));
std::string percent;
ASSERT_TRUE(getline(bt_percent, percent, ','));
sum_percent += ParseDouble(percent);
nrows++;
}
ASSERT_EQ(11, nrows);
ASSERT_EQ(100.0, sum_percent);
ASSERT_OK(env_->DeleteFile(filename));
}
}
ASSERT_OK(env_->DeleteFile(block_cache_sim_config_path_));
}
@ -366,6 +543,7 @@ TEST_F(BlockCacheTracerTest, MixedBlocks) {
// Read blocks.
BlockCacheTraceAnalyzer analyzer(trace_file_path_,
/*output_miss_ratio_curve_path=*/"",
/*compute_reuse_distance=*/true,
/*simulator=*/nullptr);
// The analyzer ends when it detects an incomplete access record.
ASSERT_EQ(Status::Incomplete(""), analyzer.Analyze());

2
trace_replay/block_cache_tracer.cc
Unescape View File

@ -29,6 +29,8 @@ bool ShouldTrace(const Slice& block_key, const TraceOptions& trace_options) {
} // namespace
const uint64_t kMicrosInSecond = 1000 * 1000;
const uint64_t kSecondInMinute = 60;
const uint64_t kSecondInHour = 3600;
const std::string BlockCacheTraceHelper::kUnknownColumnFamilyName =
"UnknownColumnFamily";
const uint64_t BlockCacheTraceHelper::kReservedGetId = 0;

3
trace_replay/block_cache_tracer.h
Unescape View File

@ -17,6 +17,9 @@
namespace rocksdb {
extern const uint64_t kMicrosInSecond;
extern const uint64_t kSecondInMinute;
extern const uint64_t kSecondInHour;
class BlockCacheTraceHelper {
public:

Write Preview
Loading…
Cancel
Save