Reduce moving memory in LDB::ScanCommand

Summary:
Based on https://github.com/facebook/rocksdb/issues/843
It looks that when the data is hot we spend significant amount of time moving data out of RocksDB blocks. This patch reduce moving memory when possible

Original performance
```
$ time ./ldb --db=/home/tec/local/ellina_test/testdb scan > /dev/null
real	0m16.736s
user	0m11.993s
sys	0m4.725s
```

Performance after reducing memcpy
```
$ time ./ldb --db=/home/tec/local/ellina_test/testdb scan > /dev/null
real	0m11.590s
user	0m6.983s
sys	0m4.595s
```

Test Plan:
dump the output of the scan into 2 files and verifying the are exactly the same
make check

Reviewers: sdong, yhchiang, anthony, rven, igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D51093
main
Islam AbdelRahman 9 years ago
parent 890f44f46e
commit 88e0527724
  1. 29
      tools/ldb_cmd.cc
  2. 11
      tools/ldb_tool.cc

@ -1762,7 +1762,6 @@ void ScanCommand::DoCommand() {
for ( ; for ( ;
it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_); it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
it->Next()) { it->Next()) {
string key = ldb_options_.key_formatter->Format(it->key());
if (is_db_ttl_) { if (is_db_ttl_) {
TtlIterator* it_ttl = dynamic_cast<TtlIterator*>(it); TtlIterator* it_ttl = dynamic_cast<TtlIterator*>(it);
assert(it_ttl); assert(it_ttl);
@ -1774,11 +1773,29 @@ void ScanCommand::DoCommand() {
fprintf(stdout, "%s ", ReadableTime(rawtime).c_str()); fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
} }
} }
string value = it->value().ToString();
fprintf(stdout, "%s : %s\n", Slice key_slice = it->key();
(is_key_hex_ ? "0x" + it->key().ToString(true) : key).c_str(), Slice val_slice = it->value();
(is_value_hex_ ? StringToHex(value) : value).c_str()
); std::string formatted_key;
if (is_key_hex_) {
formatted_key = "0x" + key_slice.ToString(true /* hex */);
key_slice = formatted_key;
} else if (ldb_options_.key_formatter) {
formatted_key = ldb_options_.key_formatter->Format(key_slice);
key_slice = formatted_key;
}
std::string formatted_value;
if (is_value_hex_) {
formatted_value = "0x" + val_slice.ToString(true /* hex */);
val_slice = formatted_value;
}
fprintf(stdout, "%.*s : %.*s\n",
static_cast<int>(key_slice.size()), key_slice.data(),
static_cast<int>(val_slice.size()), val_slice.data());
num_keys_scanned++; num_keys_scanned++;
if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) { if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) {
break; break;

@ -9,16 +9,7 @@
namespace rocksdb { namespace rocksdb {
class DefaultSliceFormatter : public SliceFormatter { LDBOptions::LDBOptions() {}
public:
virtual std::string Format(const Slice& s) const override {
return s.ToString();
}
};
LDBOptions::LDBOptions()
: key_formatter(new DefaultSliceFormatter()) {
}
class LDBCommandRunner { class LDBCommandRunner {
public: public:

Loading…
Cancel
Save