From cce702a6e44be094b7c63ca4ed9eb018ed10d58a Mon Sep 17 00:00:00 2001 From: Islam AbdelRahman Date: Thu, 25 Aug 2016 12:57:35 -0700 Subject: [PATCH] [db_bench] Support single benchmark arguments (Repeat for X times, Warm up for X times), Support CombinedStats (AVG / MEDIAN) Summary: This diff allow us to run a single benchmark X times and warm it up for Y times. and see the AVG & MEDIAN throughput of these X runs for example ``` $ ./db_bench --benchmarks="fillseq,readseq[X5-W2]" Initializing RocksDB Options from the specified file Initializing RocksDB Options from command-line flags RocksDB: version 4.12 Date: Wed Aug 24 10:45:26 2016 CPU: 32 * Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz CPUCache: 20480 KB Keys: 16 bytes each Values: 100 bytes each (50 bytes after compression) Entries: 1000000 Prefix: 0 bytes Keys per prefix: 0 RawSize: 110.6 MB (estimated) FileSize: 62.9 MB (estimated) Write rate: 0 bytes/second Compression: Snappy Memtablerep: skip_list Perf Level: 1 WARNING: Assertions are enabled; benchmarks unnecessarily slow ------------------------------------------------ Initializing RocksDB Options from the specified file Initializing RocksDB Options from command-line flags DB path: [/tmp/rocksdbtest-8616/dbbench] fillseq : 4.695 micros/op 212971 ops/sec; 23.6 MB/s DB path: [/tmp/rocksdbtest-8616/dbbench] Warming up benchmark by running 2 times readseq : 0.214 micros/op 4677005 ops/sec; 517.4 MB/s readseq : 0.212 micros/op 4706834 ops/sec; 520.7 MB/s Running benchmark for 5 times readseq : 0.218 micros/op 4588187 ops/sec; 507.6 MB/s readseq : 0.208 micros/op 4816538 ops/sec; 532.8 MB/s readseq : 0.213 micros/op 4685376 ops/sec; 518.3 MB/s readseq : 0.214 micros/op 4676787 ops/sec; 517.4 MB/s readseq : 0.217 micros/op 4618532 ops/sec; 510.9 MB/s readseq [AVG 5 runs] : 4677084 ops/sec; 517.4 MB/sec readseq [MEDIAN 5 runs] : 4676787 ops/sec; 517.4 MB/sec ``` Test Plan: run db_bench Reviewers: sdong, andrewkr, yhchiang Reviewed By: yhchiang Subscribers: andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D62235 --- tools/db_bench_tool.cc | 136 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 7 deletions(-) diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index a1ac63e69..198e2ddb6 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -1279,6 +1279,7 @@ static std::unordered_map> {kOthers, "op"} }; +class CombinedStats; class Stats { private: int id_; @@ -1296,6 +1297,7 @@ class Stats { std::string message_; bool exclude_from_merge_; ReporterAgent* reporter_agent_; // does not own + friend class CombinedStats; public: Stats() { Start(-1); } @@ -1560,6 +1562,75 @@ class Stats { } }; +class CombinedStats { + public: + void AddStats(const Stats& stat) { + uint64_t total_ops = stat.done_; + uint64_t total_bytes_ = stat.bytes_; + double elapsed; + + if (total_ops < 1) { + total_ops = 1; + } + + elapsed = (stat.finish_ - stat.start_) * 1e-6; + throughput_ops_.emplace_back(total_ops / elapsed); + + if (total_bytes_ > 0) { + double mbs = (total_bytes_ / 1048576.0); + throughput_mbs_.emplace_back(mbs / elapsed); + } + } + + void Report(const std::string& bench_name) { + const char* name = bench_name.c_str(); + int num_runs = static_cast(throughput_ops_.size()); + + if (throughput_mbs_.size() == throughput_ops_.size()) { + fprintf(stdout, + "%s [AVG %d runs] : %d ops/sec; %6.1f MB/sec\n" + "%s [MEDIAN %d runs] : %d ops/sec; %6.1f MB/sec\n", + name, num_runs, static_cast(CalcAvg(throughput_ops_)), + CalcAvg(throughput_mbs_), name, num_runs, + static_cast(CalcMedian(throughput_ops_)), + CalcMedian(throughput_mbs_)); + } else { + fprintf(stdout, + "%s [AVG %d runs] : %d ops/sec\n" + "%s [MEDIAN %d runs] : %d ops/sec\n", + name, num_runs, static_cast(CalcAvg(throughput_ops_)), name, + num_runs, static_cast(CalcMedian(throughput_ops_))); + } + } + + private: + double CalcAvg(std::vector data) { + double avg = 0; + for (double x : data) { + avg += x; + } + avg = avg / data.size(); + return avg; + } + + double CalcMedian(std::vector data) { + assert(data.size() > 0); + std::sort(data.begin(), data.end()); + + size_t mid = data.size() / 2; + if (data.size() % 2 == 1) { + // Odd number of entries + return data[mid]; + } else { + // Even number of entries + return (data[mid] + data[mid - 1]) / 2; + } + } + + std::vector throughput_ops_; + std::vector throughput_mbs_; +}; + class TimestampEmulator { private: std::atomic timestamp_; @@ -2066,6 +2137,36 @@ class Benchmark { bool fresh_db = false; int num_threads = FLAGS_threads; + int num_repeat = 1; + int num_warmup = 0; + if (!name.empty() && *name.rbegin() == ']') { + auto it = name.find('['); + if (it == std::string::npos) { + fprintf(stderr, "unknown benchmark arguments '%s'\n", name.c_str()); + exit(1); + } + std::string args = name.substr(it + 1); + args.resize(args.size() - 1); + name.resize(it); + + std::string bench_arg; + std::stringstream args_stream(args); + while (std::getline(args_stream, bench_arg, '-')) { + if (bench_arg.empty()) { + continue; + } + if (bench_arg[0] == 'X') { + // Repeat the benchmark n times + std::string num_str = bench_arg.substr(1); + num_repeat = std::stoi(num_str); + } else if (bench_arg[0] == 'W') { + // Warm up the benchmark for n times + std::string num_str = bench_arg.substr(1); + num_warmup = std::stoi(num_str); + } + } + } + if (name == "fillseq") { fresh_db = true; method = &Benchmark::WriteSeq; @@ -2227,7 +2328,26 @@ class Benchmark { if (method != nullptr) { fprintf(stdout, "DB path: [%s]\n", FLAGS_db.c_str()); - RunBenchmark(num_threads, name, method); + if (num_warmup > 0) { + printf("Warming up benchmark by running %d times\n", num_warmup); + } + + for (int i = 0; i < num_warmup; i++) { + RunBenchmark(num_threads, name, method); + } + + if (num_repeat > 1) { + printf("Running benchmark for %d times\n", num_repeat); + } + + CombinedStats combined_stats; + for (int i = 0; i < num_repeat; i++) { + Stats stats = RunBenchmark(num_threads, name, method); + combined_stats.AddStats(stats); + } + if (num_repeat > 1) { + combined_stats.Report(name); + } } if (post_process_method != nullptr) { (this->*post_process_method)(); @@ -2282,8 +2402,8 @@ class Benchmark { } } - void RunBenchmark(int n, Slice name, - void (Benchmark::*method)(ThreadState*)) { + Stats RunBenchmark(int n, Slice name, + void (Benchmark::*method)(ThreadState*)) { SharedState shared; shared.total = n; shared.num_initialized = 0; @@ -2341,6 +2461,11 @@ class Benchmark { } shared.mu.Unlock(); + for (int i = 0; i < n; i++) { + delete arg[i].thread; + } + delete[] arg; + // Stats for some threads can be excluded. Stats merge_stats; for (int i = 0; i < n; i++) { @@ -2348,10 +2473,7 @@ class Benchmark { } merge_stats.Report(name); - for (int i = 0; i < n; i++) { - delete arg[i].thread; - } - delete[] arg; + return merge_stats; } void Crc32c(ThreadState* thread) {