NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split histogram per OperationType in db_bench

Browse Source
main
SherlockNoMad 9 years ago
parent f307036bde
commit ebc2d490d1
1 changed files with 92 additions and 48 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 140
      db/db_bench.cc

140
db/db_bench.cc
Unescape View File

@ -39,6 +39,7 @@ int main() {
#include <condition_variable> #include <condition_variable>
#include <mutex> #include <mutex>
#include <thread> #include <thread>
#include <unordered_map>
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/version_set.h" #include "db/version_set.h"
@ -741,11 +742,6 @@ DEFINE_bool(identity_as_first_hash, false, "the first hash function of cuckoo "
"table becomes an identity function. This is only valid when key " "table becomes an identity function. This is only valid when key "
"is 8 bytes"); "is 8 bytes");
enum PutOrMerge {
kPut,
kMerge
};
enum RepFactory { enum RepFactory {
kSkipList, kSkipList,
kPrefixHash, kPrefixHash,
@ -1163,6 +1159,35 @@ class ReporterAgent {
bool stop_; bool stop_;
}; };
enum OperationType : unsigned char {
kRead = 0,
kWrite,
kDelete,
kSeek,
kMerge,
kUpdate,
kCompress,
kUncompress,
kCrc,
kHash,
kOthers
};
static std::unordered_map<OperationType, std::string, std::hash<unsigned char>>
OperationTypeString = {
{kRead, "read"},
{kWrite, "write"},
{kDelete, "delete"},
{kSeek, "seek"},
{kMerge, "merge"},
{kUpdate, "update"},
{kCompress, "compress"},
{kCompress, "uncompress"},
{kCrc, "crc"},
{kHash, "hash"},
{kOthers, "op"}
};
class Stats { class Stats {
private: private:
int id_; int id_;
@ -1175,7 +1200,8 @@ class Stats {
int64_t bytes_; int64_t bytes_;
double last_op_finish_; double last_op_finish_;
double last_report_finish_; double last_report_finish_;
HistogramImpl hist_; std::unordered_map<OperationType, HistogramImpl,
std::hash<unsigned char>> hist_;
std::string message_; std::string message_;
bool exclude_from_merge_; bool exclude_from_merge_;
ReporterAgent* reporter_agent_; // does not own ReporterAgent* reporter_agent_; // does not own
@ -1191,7 +1217,7 @@ class Stats {
id_ = id; id_ = id;
next_report_ = FLAGS_stats_interval ? FLAGS_stats_interval : 100; next_report_ = FLAGS_stats_interval ? FLAGS_stats_interval : 100;
last_op_finish_ = start_; last_op_finish_ = start_;
hist_.Clear(); hist_.clear();
done_ = 0; done_ = 0;
last_report_done_ = 0; last_report_done_ = 0;
bytes_ = 0; bytes_ = 0;
@ -1208,7 +1234,15 @@ class Stats {
if (other.exclude_from_merge_) if (other.exclude_from_merge_)
return; return;
hist_.Merge(other.hist_); for (auto it = other.hist_.begin(); it != other.hist_.end(); ++it) {
auto this_it = hist_.find(it->first);
if (this_it != hist_.end()) {
this_it->second.Merge(other.hist_.at(it->first));
} else {
hist_.insert({ it->first, it->second });
}
}
done_ += other.done_; done_ += other.done_;
bytes_ += other.bytes_; bytes_ += other.bytes_;
seconds_ += other.seconds_; seconds_ += other.seconds_;
@ -1261,14 +1295,22 @@ class Stats {
} }
} }
void FinishedOps(DBWithColumnFamilies* db_with_cfh, DB* db, int64_t num_ops) { void FinishedOps(DBWithColumnFamilies* db_with_cfh, DB* db, int64_t num_ops,
enum OperationType op_type = kOthers) {
if (reporter_agent_) { if (reporter_agent_) {
reporter_agent_->ReportFinishedOps(num_ops); reporter_agent_->ReportFinishedOps(num_ops);
} }
if (FLAGS_histogram) { if (FLAGS_histogram) {
double now = FLAGS_env->NowMicros(); double now = FLAGS_env->NowMicros();
double micros = now - last_op_finish_; double micros = now - last_op_finish_;
hist_.Add(micros);
if (hist_.find(op_type) == hist_.end())
{
HistogramImpl hist_temp;
hist_.insert({op_type, hist_temp});
}
hist_[op_type].Add(micros);
if (micros > 20000 && !FLAGS_stats_interval) { if (micros > 20000 && !FLAGS_stats_interval) {
fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
fflush(stderr); fflush(stderr);
@ -1397,7 +1439,11 @@ class Stats {
(extra.empty() ? "" : " "), (extra.empty() ? "" : " "),
extra.c_str()); extra.c_str());
if (FLAGS_histogram) { if (FLAGS_histogram) {
fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); for (auto it = hist_.begin(); it != hist_.end(); ++it) {
fprintf(stdout, "Microseconds per %s:\n%s\n",
OperationTypeString[it->first].c_str(),
it->second.ToString().c_str());
}
} }
if (FLAGS_report_file_operations) { if (FLAGS_report_file_operations) {
ReportFileOpEnv* env = static_cast<ReportFileOpEnv*>(FLAGS_env); ReportFileOpEnv* env = static_cast<ReportFileOpEnv*>(FLAGS_env);
@ -2138,7 +2184,7 @@ class Benchmark {
uint32_t crc = 0; uint32_t crc = 0;
while (bytes < 500 * 1048576) { while (bytes < 500 * 1048576) {
crc = crc32c::Value(data.data(), size); crc = crc32c::Value(data.data(), size);
thread->stats.FinishedOps(nullptr, nullptr, 1); thread->stats.FinishedOps(nullptr, nullptr, 1, kCrc);
bytes += size; bytes += size;
} }
// Print so result is not dead // Print so result is not dead
@ -2157,7 +2203,7 @@ class Benchmark {
unsigned int xxh32 = 0; unsigned int xxh32 = 0;
while (bytes < 500 * 1048576) { while (bytes < 500 * 1048576) {
xxh32 = XXH32(data.data(), size, 0); xxh32 = XXH32(data.data(), size, 0);
thread->stats.FinishedOps(nullptr, nullptr, 1); thread->stats.FinishedOps(nullptr, nullptr, 1, kHash);
bytes += size; bytes += size;
} }
// Print so result is not dead // Print so result is not dead
@ -2178,7 +2224,7 @@ class Benchmark {
ptr = ap.load(std::memory_order_acquire); ptr = ap.load(std::memory_order_acquire);
} }
count++; count++;
thread->stats.FinishedOps(nullptr, nullptr, 1); thread->stats.FinishedOps(nullptr, nullptr, 1, kOthers);
} }
if (ptr == nullptr) exit(1); // Disable unused variable warning. if (ptr == nullptr) exit(1); // Disable unused variable warning.
} }
@ -2196,7 +2242,7 @@ class Benchmark {
ok = CompressSlice(input, &compressed); ok = CompressSlice(input, &compressed);
produced += compressed.size(); produced += compressed.size();
bytes += input.size(); bytes += input.size();
thread->stats.FinishedOps(nullptr, nullptr, 1); thread->stats.FinishedOps(nullptr, nullptr, 1, kCompress);
} }
if (!ok) { if (!ok) {
@ -2257,7 +2303,7 @@ class Benchmark {
} }
delete[] uncompressed; delete[] uncompressed;
bytes += input.size(); bytes += input.size();
thread->stats.FinishedOps(nullptr, nullptr, 1); thread->stats.FinishedOps(nullptr, nullptr, 1, kUncompress);
} }
if (!ok) { if (!ok) {
@ -2788,7 +2834,7 @@ class Benchmark {
} }
s = db_with_cfh->db->Write(write_options_, &batch); s = db_with_cfh->db->Write(write_options_, &batch);
thread->stats.FinishedOps(db_with_cfh, db_with_cfh->db, thread->stats.FinishedOps(db_with_cfh, db_with_cfh->db,
entries_per_batch_); entries_per_batch_, kWrite);
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "put error: %s\n", s.ToString().c_str()); fprintf(stderr, "put error: %s\n", s.ToString().c_str());
exit(1); exit(1);
@ -2816,7 +2862,7 @@ class Benchmark {
int64_t bytes = 0; int64_t bytes = 0;
for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) { for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
bytes += iter->key().size() + iter->value().size(); bytes += iter->key().size() + iter->value().size();
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kRead);
++i; ++i;
} }
delete iter; delete iter;
@ -2839,7 +2885,7 @@ class Benchmark {
int64_t bytes = 0; int64_t bytes = 0;
for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) { for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
bytes += iter->key().size() + iter->value().size(); bytes += iter->key().size() + iter->value().size();
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kRead);
++i; ++i;
} }
delete iter; delete iter;
@ -2879,7 +2925,7 @@ class Benchmark {
++nonexist; ++nonexist;
} }
} }
thread->stats.FinishedOps(nullptr, db, 100); thread->stats.FinishedOps(nullptr, db, 100, kRead);
} while (!duration.Done(100)); } while (!duration.Done(100));
char msg[100]; char msg[100];
@ -2947,7 +2993,7 @@ class Benchmark {
fprintf(stderr, "Get returned an error: %s\n", s.ToString().c_str()); fprintf(stderr, "Get returned an error: %s\n", s.ToString().c_str());
abort(); abort();
} }
thread->stats.FinishedOps(db_with_cfh, db_with_cfh->db, 1); thread->stats.FinishedOps(db_with_cfh, db_with_cfh->db, 1, kRead);
} }
char msg[100]; char msg[100];
@ -2995,7 +3041,7 @@ class Benchmark {
abort(); abort();
} }
} }
thread->stats.FinishedOps(nullptr, db, entries_per_batch_); thread->stats.FinishedOps(nullptr, db, entries_per_batch_, kRead);
} }
char msg[100]; char msg[100];
@ -3011,7 +3057,7 @@ class Benchmark {
DB* db = SelectDB(thread); DB* db = SelectDB(thread);
Iterator* iter = db->NewIterator(options); Iterator* iter = db->NewIterator(options);
delete iter; delete iter;
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kOthers);
} }
} }
@ -3019,7 +3065,7 @@ class Benchmark {
if (thread->tid > 0) { if (thread->tid > 0) {
IteratorCreation(thread); IteratorCreation(thread);
} else { } else {
BGWriter(thread, kPut); BGWriter(thread, kWrite);
} }
} }
@ -3088,7 +3134,7 @@ class Benchmark {
assert(iter_to_use->status().ok()); assert(iter_to_use->status().ok());
} }
thread->stats.FinishedOps(&db_, db_.db, 1); thread->stats.FinishedOps(&db_, db_.db, 1, kSeek);
} }
delete single_iter; delete single_iter;
for (auto iter : multi_iters) { for (auto iter : multi_iters) {
@ -3109,7 +3155,7 @@ class Benchmark {
if (thread->tid > 0) { if (thread->tid > 0) {
SeekRandom(thread); SeekRandom(thread);
} else { } else {
BGWriter(thread, kPut); BGWriter(thread, kWrite);
} }
} }
@ -3137,7 +3183,7 @@ class Benchmark {
batch.Delete(key); batch.Delete(key);
} }
auto s = db->Write(write_options_, &batch); auto s = db->Write(write_options_, &batch);
thread->stats.FinishedOps(nullptr, db, entries_per_batch_); thread->stats.FinishedOps(nullptr, db, entries_per_batch_, kDelete);
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "del error: %s\n", s.ToString().c_str()); fprintf(stderr, "del error: %s\n", s.ToString().c_str());
exit(1); exit(1);
@ -3158,7 +3204,7 @@ class Benchmark {
if (thread->tid > 0) { if (thread->tid > 0) {
ReadRandom(thread); ReadRandom(thread);
} else { } else {
BGWriter(thread, kPut); BGWriter(thread, kWrite);
} }
} }
@ -3170,7 +3216,7 @@ class Benchmark {
} }
} }
void BGWriter(ThreadState* thread, enum PutOrMerge write_merge) { void BGWriter(ThreadState* thread, enum OperationType write_merge) {
// Special thread that keeps writing until other threads are done. // Special thread that keeps writing until other threads are done.
RandomGenerator gen; RandomGenerator gen;
double last = FLAGS_env->NowMicros(); double last = FLAGS_env->NowMicros();
@ -3203,7 +3249,7 @@ class Benchmark {
GenerateKeyFromInt(thread->rand.Next() % FLAGS_num, FLAGS_num, &key); GenerateKeyFromInt(thread->rand.Next() % FLAGS_num, FLAGS_num, &key);
Status s; Status s;
if (write_merge == kPut) { if (write_merge == kWrite) {
s = db->Put(write_options_, key, gen.Generate(value_size_)); s = db->Put(write_options_, key, gen.Generate(value_size_));
} else { } else {
s = db->Merge(write_options_, key, gen.Generate(value_size_)); s = db->Merge(write_options_, key, gen.Generate(value_size_));
@ -3214,7 +3260,7 @@ class Benchmark {
exit(1); exit(1);
} }
bytes += key.size() + value_size_; bytes += key.size() + value_size_;
thread->stats.FinishedOps(&db_, db_.db, 1); thread->stats.FinishedOps(&db_, db_.db, 1, kWrite);
++num_writes; ++num_writes;
if (writes_per_second_by_10 && num_writes >= writes_per_second_by_10) { if (writes_per_second_by_10 && num_writes >= writes_per_second_by_10) {
@ -3355,6 +3401,7 @@ class Benchmark {
} }
get_weight--; get_weight--;
gets_done++; gets_done++;
thread->stats.FinishedOps(&db_, db_.db, 1, kRead);
} else if (put_weight > 0) { } else if (put_weight > 0) {
// then do all the corresponding number of puts // then do all the corresponding number of puts
// for all the gets we have done earlier // for all the gets we have done earlier
@ -3365,6 +3412,7 @@ class Benchmark {
} }
put_weight--; put_weight--;
puts_done++; puts_done++;
thread->stats.FinishedOps(&db_, db_.db, 1, kWrite);
} else if (delete_weight > 0) { } else if (delete_weight > 0) {
Status s = DeleteMany(db, write_options_, key); Status s = DeleteMany(db, write_options_, key);
if (!s.ok()) { if (!s.ok()) {
@ -3373,9 +3421,8 @@ class Benchmark {
} }
delete_weight--; delete_weight--;
deletes_done++; deletes_done++;
thread->stats.FinishedOps(&db_, db_.db, 1, kDelete);
} }
thread->stats.FinishedOps(&db_, db_.db, 1);
} }
char msg[100]; char msg[100];
snprintf(msg, sizeof(msg), snprintf(msg, sizeof(msg),
@ -3422,6 +3469,7 @@ class Benchmark {
} }
get_weight--; get_weight--;
reads_done++; reads_done++;
thread->stats.FinishedOps(nullptr, db, 1, kRead);
} else if (put_weight > 0) { } else if (put_weight > 0) {
// then do all the corresponding number of puts // then do all the corresponding number of puts
// for all the gets we have done earlier // for all the gets we have done earlier
@ -3432,8 +3480,8 @@ class Benchmark {
} }
put_weight--; put_weight--;
writes_done++; writes_done++;
thread->stats.FinishedOps(nullptr, db, 1, kWrite);
} }
thread->stats.FinishedOps(nullptr, db, 1);
} }
char msg[100]; char msg[100];
snprintf(msg, sizeof(msg), "( reads:%" PRIu64 " writes:%" PRIu64 \ snprintf(msg, sizeof(msg), "( reads:%" PRIu64 " writes:%" PRIu64 \
@ -3475,7 +3523,7 @@ class Benchmark {
exit(1); exit(1);
} }
bytes += key.size() + value_size_; bytes += key.size() + value_size_;
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kUpdate);
} }
char msg[100]; char msg[100];
snprintf(msg, sizeof(msg), snprintf(msg, sizeof(msg),
@ -3530,7 +3578,7 @@ class Benchmark {
exit(1); exit(1);
} }
bytes += key.size() + value.size(); bytes += key.size() + value.size();
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kUpdate);
} }
char msg[100]; char msg[100];
@ -3568,7 +3616,7 @@ class Benchmark {
exit(1); exit(1);
} }
bytes += key.size() + value_size_; bytes += key.size() + value_size_;
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kMerge);
} }
// Print some statistics // Print some statistics
@ -3610,9 +3658,8 @@ class Benchmark {
fprintf(stderr, "merge error: %s\n", s.ToString().c_str()); fprintf(stderr, "merge error: %s\n", s.ToString().c_str());
exit(1); exit(1);
} }
num_merges++; num_merges++;
thread->stats.FinishedOps(nullptr, db, 1, kMerge);
} else { } else {
Status s = db->Get(options, key, &value); Status s = db->Get(options, key, &value);
if (value.length() > max_length) if (value.length() > max_length)
@ -3625,12 +3672,9 @@ class Benchmark {
} else if (!s.IsNotFound()) { } else if (!s.IsNotFound()) {
num_hits++; num_hits++;
} }
num_gets++; num_gets++;
thread->stats.FinishedOps(nullptr, db, 1, kRead);
} }
thread->stats.FinishedOps(nullptr, db, 1);
} }
char msg[100]; char msg[100];
@ -3657,7 +3701,7 @@ class Benchmark {
GenerateKeyFromInt(i, FLAGS_num, &key); GenerateKeyFromInt(i, FLAGS_num, &key);
iter->Seek(key); iter->Seek(key);
assert(iter->Valid() && iter->key() == key); assert(iter->Valid() && iter->key() == key);
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kSeek);
for (int j = 0; j < FLAGS_seek_nexts && i + 1 < FLAGS_num; ++j) { for (int j = 0; j < FLAGS_seek_nexts && i + 1 < FLAGS_num; ++j) {
if (!FLAGS_reverse_iterator) { if (!FLAGS_reverse_iterator) {
@ -3667,12 +3711,12 @@ class Benchmark {
} }
GenerateKeyFromInt(++i, FLAGS_num, &key); GenerateKeyFromInt(++i, FLAGS_num, &key);
assert(iter->Valid() && iter->key() == key); assert(iter->Valid() && iter->key() == key);
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kSeek);
} }
iter->Seek(key); iter->Seek(key);
assert(iter->Valid() && iter->key() == key); assert(iter->Valid() && iter->key() == key);
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kSeek);
} }
} }
@ -3834,7 +3878,7 @@ class Benchmark {
} }
if (!failed) { if (!failed) {
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kOthers);
} }
transactions_done++; transactions_done++;
@ -3960,7 +4004,7 @@ class Benchmark {
exit(1); exit(1);
} }
thread->stats.FinishedOps(nullptr, db, 1); thread->stats.FinishedOps(nullptr, db, 1, kOthers);
} }
char msg[200]; char msg[200];

Write Preview
Loading…
Cancel
Save