NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Commandline tool to compace LevelDB databases.

Browse Source 
Summary:
A simple CLI which calles DB->CompactRange()
Can take String key's as range.

Test Plan:
Inserted data into a table.
Waited for a minute, used compact tool on it. File modification time's
changed so Compact did something on the files.

Existing unit tests work.

Reviewers: heyongqiang, dhruba

Reviewed By: dhruba

Differential Revision: https://reviews.facebook.net/D5697
main
Abhishek Kona 12 years ago
parent a321d5be9e
commit fec81318b0
2 changed files with 153 additions and 65 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 16
      db/db_bench.cc
  2. 202
      tools/ldb.cc

16
db/db_bench.cc
Unescape View File

@ -209,7 +209,7 @@ class RandomGenerator {
pos_ += len;
return Slice(data_.data() + pos_ - len, len);
}
};
};
static Slice TrimSpace(Slice s) {
int start = 0;
while (start < s.size() && isspace(s[start])) {
@ -515,9 +515,9 @@ class Benchmark {
public:
Benchmark()
: cache_(FLAGS_cache_size >= 0 ?
(FLAGS_cache_numshardbits >= 1 ?
NewLRUCache(FLAGS_cache_size, FLAGS_cache_numshardbits) :
: cache_(FLAGS_cache_size >= 0 ?
(FLAGS_cache_numshardbits >= 1 ?
NewLRUCache(FLAGS_cache_size, FLAGS_cache_numshardbits) :
NewLRUCache(FLAGS_cache_size)) : NULL),
filter_policy_(FLAGS_bloom_bits >= 0
? NewBloomFilterPolicy(FLAGS_bloom_bits)
@ -528,9 +528,9 @@ class Benchmark {
entries_per_batch_(1),
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
writes_(FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes),
readwrites_((FLAGS_writes < 0 && FLAGS_reads < 0)? FLAGS_num :
readwrites_((FLAGS_writes < 0 && FLAGS_reads < 0)? FLAGS_num :
((FLAGS_writes > FLAGS_reads) ? FLAGS_writes : FLAGS_reads)
),
),
heap_counter_(0) {
std::vector<std::string> files;
FLAGS_env->GetChildren(FLAGS_db, &files);
@ -1048,7 +1048,7 @@ class Benchmark {
//
// This is diffferent from ReadWhileWriting because it does not use
// an extra thread.
// an extra thread.
//
void ReadRandomWriteRandom(ThreadState* thread) {
ReadOptions options(FLAGS_verify_checksum, true);
@ -1090,7 +1090,7 @@ class Benchmark {
thread->stats.FinishedSingleOp();
}
char msg[100];
snprintf(msg, sizeof(msg), "( reads:%ld writes:%ld total:%ld )",
snprintf(msg, sizeof(msg), "( reads:%ld writes:%ld total:%ld )",
reads_done, writes_done, readwrites_);
thread->stats.AddMessage(msg);
}

202
tools/ldb.cc
Unescape View File

@ -7,9 +7,11 @@
#include <sstream>
#include "leveldb/db.h"
#include "leveldb/options.h"
#include "leveldb/iterator.h"
#include "leveldb/slice.h"
std::string HexToString(std::string str) {
std::string HexToString(const std::string& str) {
std::string parsed;
for (int i = 0; i < str.length(); ) {
int c;
@ -20,73 +22,58 @@ std::string HexToString(std::string str) {
return parsed;
}
static void print_help() {
static void print_usage() {
fprintf(stderr,
"db_dump "
"--start=[START_KEY] "
"--end=[END_KEY] "
"--max_keys=[NUM] "
"--hex "
"--count_only "
"--stats "
"[PATH]\n");
"ldb [compact|dump] "
"--db-path=database_path "
"[--from=START KEY] "
"[--to=END KEY ] "
"[--max_keys=[NUM] (only for dump)] "
"[--hex ] "
"[--count_only (only for dump)] "
"[--stats (only for dump) ] \n");
}
int main(int argc, char** argv) {
std::string db_path;
std::string start;
std::string end;
uint64_t max_keys = -1;
bool print_stats = false;
bool count_only = false;
uint64_t count = 0;
bool hex = false;
// Parse command line args
char junk;
uint64_t n;
for (int i = 1; i < argc; i++) {
if (strncmp(argv[i], "--start=", 8) == 0) {
start= argv[i] + 8;
} else if (strncmp(argv[i], "--end=", 6) == 0) {
end = argv[i] + 6;
} else if (sscanf(argv[i], "--max_keys=%ld%c", &n, &junk) == 1) {
max_keys = n;
} else if (strncmp(argv[i], "--stats", 7) == 0) {
print_stats = true;
} else if (strncmp(argv[i], "--count_only", 12) == 0) {
count_only = true;
} else if (strncmp(argv[i], "--hex", 5) == 0) {
hex = true;
} else if (i == (argc - 1)) {
db_path = argv[i];
} else {
print_help();
exit(1);
}
static void safe_open_db(const std::string& dbname, leveldb::DB** db) {
leveldb::Options options;
options.create_if_missing = false;
leveldb::Status status = leveldb::DB::Open(options, dbname, db);
if(!status.ok()) {
fprintf(
stderr,
"Could not open db at %s\nERROR: %s",
dbname.data(),
status.ToString().data()
);
exit(1);
}
}
static void dump_db(
const std::string& db_path,
std::string& start,
std::string& end,
int64_t max_keys,
const bool hex,
const bool print_stats,
const bool count_only
) {
// Parse command line args
uint64_t count = 0;
if (hex) {
start = HexToString(start);
end = HexToString(end);
}
if (db_path.empty()) {
print_help();
exit(1);
}
// Open DB
leveldb::Options options;
leveldb::DB *db;
leveldb::Status status = leveldb::DB::Open(options, db_path, &db);
safe_open_db(db_path, &db);
if (!status.ok()) {
fprintf(stderr, "%s\n", status.ToString().c_str());
exit(1);
}
// Print DB stats if desired
if (print_stats) {
std::string stats;
if (db->GetProperty("leveldb.stats", &stats)) {
@ -96,7 +83,7 @@ int main(int argc, char** argv) {
// Setup key iterator
leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions());
status = iter->status();
leveldb::Status status = iter->status();
if (!status.ok()) {
fprintf(stderr, "%s\n", status.ToString().c_str());
delete db;
@ -140,7 +127,108 @@ int main(int argc, char** argv) {
// Clean up
delete iter;
delete db;
}
return 0;
static void compact(
const std::string dbname,
std::string from,
std::string to,
const bool hex
) {
leveldb::DB* db;
safe_open_db(dbname, &db);
if(hex) {
from = HexToString(from);
to = HexToString(to);
}
leveldb::Slice* begin = from.empty() ? NULL : new leveldb::Slice(from);
leveldb::Slice* end = to.empty() ? NULL : new leveldb::Slice(to);
db->CompactRange(begin, end);
delete db;
}
int main(int argc, char** argv) {
enum {
DUMP, COMPACT
} command;
if (argc < 2) {
print_usage();
exit(1);
}
size_t n;
const std::string dbnameKey = "--db-path=";
const std::string toKey = "--to=";
const std::string fromKey = "--from=";
std::string dbname;
bool dbnameFound = false;
std::string from;
std::string to;
int64_t temp;
int64_t max_keys = -1;
bool print_stats = false;
bool count_only = false;
bool hex = false;
char junk;
std::string commandString = argv[1];
if (commandString == "dump") {
command = DUMP;
} else if (commandString == "compact") {
command = COMPACT;
} else {
print_usage();
exit(1);
}
for (int i = 2; i < argc; i++) {
std::string param(argv[i]);
if ((n = param.find(dbnameKey)) != std::string::npos) {
dbname = param.substr(dbnameKey.size());
dbnameFound = true;
} else if ((n = param.find(fromKey)) != std::string::npos) {
from = param.substr(fromKey.size());
} else if ((n = param.find(toKey)) != std::string::npos) {
to = param.substr(toKey.size());
} else if (sscanf(argv[i], "--max_keys=%ld%c", &temp, &junk) == 1) {
max_keys = temp;
} else if (strncmp(argv[i], "--stats", 7) == 0) {
print_stats = true;
} else if (strncmp(argv[i], "--count_only", 12) == 0) {
count_only = true;
} else if (strncmp(argv[i], "--hex", 5) == 0) {
hex = true;
} else {
print_usage();
exit(1);
}
}
if (!dbnameFound || dbname.empty()) {
fprintf(stderr, "DB path required. See help\n");
print_usage();
exit(1);
}
switch(command) {
case DUMP:
dump_db(dbname, from, to, max_keys, hex, print_stats, count_only);
break;
case COMPACT:
compact(dbname, from, to, hex);
break;
default:
print_usage();
exit(1);
}
return 0;
}

Write Preview
Loading…
Cancel
Save