NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enable the ability to set key size in db_bench in rocksdb

Browse Source 
Summary:
1. the default value for key size is still 16
2. enable the ability to set the key size via command line --key_size=

Test Plan:
build & run db_banch and pass some value via command line.
verify it works correctly.

Reviewers: sheki

Reviewed By: sheki

CC: leveldb

Differential Revision: https://reviews.facebook.net/D8943
main
bil 12 years ago
parent ec96ad5405
commit 4992633751
1 changed files with 58 additions and 42 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 100
      db/db_bench.cc

100
db/db_bench.cc
Unescape View File

@ -65,7 +65,8 @@ static const char* FLAGS_benchmarks =
"snappyuncomp," "snappyuncomp,"
"acquireload," "acquireload,"
; ;
// the maximum size of key in bytes
static const int MAX_KEY_SIZE = 128;
// Number of key/values to place in database // Number of key/values to place in database
static long FLAGS_num = 1000000; static long FLAGS_num = 1000000;
@ -89,6 +90,9 @@ static int FLAGS_threads = 1;
// Size of each value // Size of each value
static int FLAGS_value_size = 100; static int FLAGS_value_size = 100;
//size of each key
static int FLAGS_key_size = 16;
// Arrange to generate values that shrink to this fraction of // Arrange to generate values that shrink to this fraction of
// their original size after compression // their original size after compression
static double FLAGS_compression_ratio = 0.5; static double FLAGS_compression_ratio = 0.5;
@ -485,26 +489,26 @@ class Benchmark {
DB* db_; DB* db_;
long num_; long num_;
int value_size_; int value_size_;
int key_size_;
int entries_per_batch_; int entries_per_batch_;
WriteOptions write_options_; WriteOptions write_options_;
long reads_; long reads_;
long writes_; long writes_;
long readwrites_; long readwrites_;
int heap_counter_; int heap_counter_;
char keyFormat_[100]; // this string will contain the format of key. e.g "%016d"
void PrintHeader() { void PrintHeader() {
const int kKeySize = 16;
PrintEnvironment(); PrintEnvironment();
fprintf(stdout, "Keys: %d bytes each\n", kKeySize); fprintf(stdout, "Keys: %d bytes each\n", FLAGS_key_size);
fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n", fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
FLAGS_value_size, FLAGS_value_size,
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
fprintf(stdout, "Entries: %ld\n", num_); fprintf(stdout, "Entries: %ld\n", num_);
fprintf(stdout, "RawSize: %.1f MB (estimated)\n", fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) ((static_cast<int64_t>(FLAGS_key_size + FLAGS_value_size) * num_)
/ 1048576.0)); / 1048576.0));
fprintf(stdout, "FileSize: %.1f MB (estimated)\n", fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) (((FLAGS_key_size + FLAGS_value_size * FLAGS_compression_ratio) * num_)
/ 1048576.0)); / 1048576.0));
switch (FLAGS_compression_type) { switch (FLAGS_compression_type) {
@ -655,6 +659,7 @@ class Benchmark {
db_(nullptr), db_(nullptr),
num_(FLAGS_num), num_(FLAGS_num),
value_size_(FLAGS_value_size), value_size_(FLAGS_value_size),
key_size_(FLAGS_key_size),
entries_per_batch_(1), entries_per_batch_(1),
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
writes_(FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes), writes_(FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes),
@ -678,7 +683,20 @@ class Benchmark {
delete db_; delete db_;
delete filter_policy_; delete filter_policy_;
} }
//this function will construct string format for key. e.g "%016d"
void ConstructStrFormatForKey(char* str, int keySize)
{
str[0] = '%';
str[1] = '0';
sprintf(str+2, "%dd", keySize);
}
unique_ptr<char []> GenerateKeyFromInt(int v)
{
unique_ptr<char []> keyInStr(new char[MAX_KEY_SIZE]);
snprintf(keyInStr.get(), MAX_KEY_SIZE, keyFormat_, v);
return keyInStr;
}
void Run() { void Run() {
PrintHeader(); PrintHeader();
Open(); Open();
@ -700,12 +718,13 @@ class Benchmark {
reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
writes_ = (FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes); writes_ = (FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes);
value_size_ = FLAGS_value_size; value_size_ = FLAGS_value_size;
key_size_ = FLAGS_key_size;
ConstructStrFormatForKey(keyFormat_, key_size_);
entries_per_batch_ = 1; entries_per_batch_ = 1;
write_options_ = WriteOptions(); write_options_ = WriteOptions();
if (FLAGS_sync) { if (FLAGS_sync) {
write_options_.sync = true; write_options_.sync = true;
} }
write_options_.disableWAL = FLAGS_disable_wal; write_options_.disableWAL = FLAGS_disable_wal;
void (Benchmark::*method)(ThreadState*) = nullptr; void (Benchmark::*method)(ThreadState*) = nullptr;
@ -1053,10 +1072,9 @@ class Benchmark {
batch.Clear(); batch.Clear();
for (int j = 0; j < entries_per_batch_; j++) { for (int j = 0; j < entries_per_batch_; j++) {
const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
char key[100]; unique_ptr<char []> key = GenerateKeyFromInt(k);
snprintf(key, sizeof(key), "%016d", k); batch.Put(key.get(), gen.Generate(value_size_));
batch.Put(key, gen.Generate(value_size_)); bytes += value_size_ + strlen(key.get());
bytes += value_size_ + strlen(key);
thread->stats.FinishedSingleOp(db_); thread->stats.FinishedSingleOp(db_);
} }
s = db_->Write(write_options_, &batch); s = db_->Write(write_options_, &batch);
@ -1101,16 +1119,14 @@ class Benchmark {
std::string value; std::string value;
long found = 0; long found = 0;
for (long i = 0; i < reads_; i++) { for (long i = 0; i < reads_; i++) {
char key[100];
const int k = thread->rand.Next() % FLAGS_num; const int k = thread->rand.Next() % FLAGS_num;
snprintf(key, sizeof(key), "%016d", k); unique_ptr<char []> key = GenerateKeyFromInt(k);
if (FLAGS_read_range < 2) { if (FLAGS_read_range < 2) {
if (db_->Get(options, key, &value).ok()) { if (db_->Get(options, key.get(), &value).ok()) {
found++; found++;
} }
} else { } else {
Slice skey(key); Slice skey(key.get());
int count = 1; int count = 1;
for (iter->Seek(skey); for (iter->Seek(skey);
iter->Valid() && count <= FLAGS_read_range; iter->Valid() && count <= FLAGS_read_range;
@ -1131,10 +1147,9 @@ class Benchmark {
ReadOptions options(FLAGS_verify_checksum, true); ReadOptions options(FLAGS_verify_checksum, true);
std::string value; std::string value;
for (long i = 0; i < reads_; i++) { for (long i = 0; i < reads_; i++) {
char key[100];
const int k = thread->rand.Next() % FLAGS_num; const int k = thread->rand.Next() % FLAGS_num;
snprintf(key, sizeof(key), "%016d.", k); unique_ptr<char []> key = GenerateKeyFromInt(k);
db_->Get(options, key, &value); db_->Get(options, key.get(), &value);
thread->stats.FinishedSingleOp(db_); thread->stats.FinishedSingleOp(db_);
} }
} }
@ -1144,10 +1159,9 @@ class Benchmark {
std::string value; std::string value;
const long range = (FLAGS_num + 99) / 100; const long range = (FLAGS_num + 99) / 100;
for (long i = 0; i < reads_; i++) { for (long i = 0; i < reads_; i++) {
char key[100];
const int k = thread->rand.Next() % range; const int k = thread->rand.Next() % range;
snprintf(key, sizeof(key), "%016d", k); unique_ptr<char []> key = GenerateKeyFromInt(k);
db_->Get(options, key, &value); db_->Get(options, key.get(), &value);
thread->stats.FinishedSingleOp(db_); thread->stats.FinishedSingleOp(db_);
} }
} }
@ -1158,11 +1172,10 @@ class Benchmark {
long found = 0; long found = 0;
for (long i = 0; i < reads_; i++) { for (long i = 0; i < reads_; i++) {
Iterator* iter = db_->NewIterator(options); Iterator* iter = db_->NewIterator(options);
char key[100];
const int k = thread->rand.Next() % FLAGS_num; const int k = thread->rand.Next() % FLAGS_num;
snprintf(key, sizeof(key), "%016d", k); unique_ptr<char []> key = GenerateKeyFromInt(k);
iter->Seek(key); iter->Seek(key.get());
if (iter->Valid() && iter->key() == key) found++; if (iter->Valid() && iter->key() == key.get()) found++;
delete iter; delete iter;
thread->stats.FinishedSingleOp(db_); thread->stats.FinishedSingleOp(db_);
} }
@ -1179,9 +1192,8 @@ class Benchmark {
batch.Clear(); batch.Clear();
for (int j = 0; j < entries_per_batch_; j++) { for (int j = 0; j < entries_per_batch_; j++) {
const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
char key[100]; unique_ptr<char []> key = GenerateKeyFromInt(k);
snprintf(key, sizeof(key), "%016d", k); batch.Delete(key.get());
batch.Delete(key);
thread->stats.FinishedSingleOp(db_); thread->stats.FinishedSingleOp(db_);
} }
s = db_->Write(write_options_, &batch); s = db_->Write(write_options_, &batch);
@ -1216,9 +1228,8 @@ class Benchmark {
} }
const int k = thread->rand.Next() % FLAGS_num; const int k = thread->rand.Next() % FLAGS_num;
char key[100]; unique_ptr<char []> key = GenerateKeyFromInt(k);
snprintf(key, sizeof(key), "%016d", k); Status s = db_->Put(write_options_, key.get(), gen.Generate(value_size_));
Status s = db_->Put(write_options_, key, gen.Generate(value_size_));
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "put error: %s\n", s.ToString().c_str()); fprintf(stderr, "put error: %s\n", s.ToString().c_str());
exit(1); exit(1);
@ -1326,9 +1337,8 @@ class Benchmark {
long deletes_done = 0; long deletes_done = 0;
// the number of iterations is the larger of read_ or write_ // the number of iterations is the larger of read_ or write_
for (long i = 0; i < readwrites_; i++) { for (long i = 0; i < readwrites_; i++) {
char key[100];
const int k = thread->rand.Next() % (FLAGS_numdistinct); const int k = thread->rand.Next() % (FLAGS_numdistinct);
snprintf(key, sizeof(key), "%016d", k); unique_ptr<char []> key = GenerateKeyFromInt(k);
if (get_weight == 0 && put_weight == 0 && delete_weight == 0) { if (get_weight == 0 && put_weight == 0 && delete_weight == 0) {
// one batch complated, reinitialize for next batch // one batch complated, reinitialize for next batch
get_weight = FLAGS_readwritepercent; get_weight = FLAGS_readwritepercent;
@ -1337,7 +1347,7 @@ class Benchmark {
} }
if (get_weight > 0) { if (get_weight > 0) {
// do all the gets first // do all the gets first
Status s = MultiGet(options, key, &value); Status s = MultiGet(options, key.get(), &value);
if (!s.ok() && !s.IsNotFound()) { if (!s.ok() && !s.IsNotFound()) {
fprintf(stderr, "get error: %s\n", s.ToString().c_str()); fprintf(stderr, "get error: %s\n", s.ToString().c_str());
// we continue after error rather than exiting so that we can // we continue after error rather than exiting so that we can
@ -1350,7 +1360,7 @@ class Benchmark {
} else if (put_weight > 0) { } else if (put_weight > 0) {
// then do all the corresponding number of puts // then do all the corresponding number of puts
// for all the gets we have done earlier // for all the gets we have done earlier
Status s = MultiPut(write_options_, key, gen.Generate(value_size_)); Status s = MultiPut(write_options_, key.get(), gen.Generate(value_size_));
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "multiput error: %s\n", s.ToString().c_str()); fprintf(stderr, "multiput error: %s\n", s.ToString().c_str());
exit(1); exit(1);
@ -1358,7 +1368,7 @@ class Benchmark {
put_weight--; put_weight--;
puts_done++; puts_done++;
} else if (delete_weight > 0) { } else if (delete_weight > 0) {
Status s = MultiDelete(write_options_, key); Status s = MultiDelete(write_options_, key.get());
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "multidelete error: %s\n", s.ToString().c_str()); fprintf(stderr, "multidelete error: %s\n", s.ToString().c_str());
exit(1); exit(1);
@ -1390,9 +1400,8 @@ class Benchmark {
long writes_done = 0; long writes_done = 0;
// the number of iterations is the larger of read_ or write_ // the number of iterations is the larger of read_ or write_
for (long i = 0; i < readwrites_; i++) { for (long i = 0; i < readwrites_; i++) {
char key[100];
const int k = thread->rand.Next() % FLAGS_num; const int k = thread->rand.Next() % FLAGS_num;
snprintf(key, sizeof(key), "%016d", k); unique_ptr<char []> key = GenerateKeyFromInt(k);
if (get_weight == 0 && put_weight == 0) { if (get_weight == 0 && put_weight == 0) {
// one batch complated, reinitialize for next batch // one batch complated, reinitialize for next batch
get_weight = FLAGS_readwritepercent; get_weight = FLAGS_readwritepercent;
@ -1400,7 +1409,7 @@ class Benchmark {
} }
if (get_weight > 0) { if (get_weight > 0) {
// do all the gets first // do all the gets first
Status s = db_->Get(options, key, &value); Status s = db_->Get(options, key.get(), &value);
if (!s.ok() && !s.IsNotFound()) { if (!s.ok() && !s.IsNotFound()) {
fprintf(stderr, "get error: %s\n", s.ToString().c_str()); fprintf(stderr, "get error: %s\n", s.ToString().c_str());
// we continue after error rather than exiting so that we can // we continue after error rather than exiting so that we can
@ -1409,7 +1418,7 @@ class Benchmark {
found++; found++;
} }
if (db_->Get(options, key, &value).ok()) { if (db_->Get(options, key.get(), &value).ok()) {
found++; found++;
} }
get_weight--; get_weight--;
@ -1417,7 +1426,7 @@ class Benchmark {
} else if (put_weight > 0) { } else if (put_weight > 0) {
// then do all the corresponding number of puts // then do all the corresponding number of puts
// for all the gets we have done earlier // for all the gets we have done earlier
Status s = db_->Put(write_options_, key, gen.Generate(value_size_)); Status s = db_->Put(write_options_, key.get(), gen.Generate(value_size_));
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "put error: %s\n", s.ToString().c_str()); fprintf(stderr, "put error: %s\n", s.ToString().c_str());
exit(1); exit(1);
@ -1508,6 +1517,13 @@ int main(int argc, char** argv) {
FLAGS_threads = n; FLAGS_threads = n;
} else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
FLAGS_value_size = n; FLAGS_value_size = n;
} else if (sscanf(argv[i], "--key_size=%d%c", &n, &junk) == 1) {
if (MAX_KEY_SIZE < n) {
fprintf(stderr, "key_size should not be larger than %d\n", MAX_KEY_SIZE);
exit(1);
} else {
FLAGS_key_size = n;
}
} else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) { } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
FLAGS_write_buffer_size = n; FLAGS_write_buffer_size = n;
} else if (sscanf(argv[i], "--max_write_buffer_number=%d%c", &n, &junk) == 1) { } else if (sscanf(argv[i], "--max_write_buffer_number=%d%c", &n, &junk) == 1) {

Write Preview
Loading…
Cancel
Save