Praveen Rao
9 years ago
commit
eb24178553
@ -1,6 +1,7 @@ |
||||
column_families_example |
||||
simple_example |
||||
c_simple_example |
||||
column_families_example |
||||
compact_files_example |
||||
transaction_example |
||||
compaction_filter_example |
||||
optimistic_transaction_example |
||||
simple_example |
||||
transaction_example |
||||
|
||||
@ -0,0 +1,84 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <rocksdb/compaction_filter.h> |
||||
#include <rocksdb/db.h> |
||||
#include <rocksdb/merge_operator.h> |
||||
#include <rocksdb/options.h> |
||||
|
||||
class MyMerge : public rocksdb::MergeOperator { |
||||
public: |
||||
bool FullMerge(const rocksdb::Slice& key, |
||||
const rocksdb::Slice* existing_value, |
||||
const std::deque<std::string>& operand_list, |
||||
std::string* new_value, |
||||
rocksdb::Logger* logger) const override { |
||||
new_value->clear(); |
||||
if (existing_value != nullptr) { |
||||
new_value->assign(existing_value->data(), existing_value->size()); |
||||
} |
||||
for (const std::string& m : operand_list) { |
||||
fprintf(stderr, "Merge(%s)\n", m.c_str()); |
||||
assert(m != "bad"); // the compaction filter filters out bad values
|
||||
new_value->assign(m); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
const char* Name() const override { return "MyMerge"; } |
||||
}; |
||||
|
||||
class MyFilter : public rocksdb::CompactionFilter { |
||||
public: |
||||
bool Filter(int level, const rocksdb::Slice& key, |
||||
const rocksdb::Slice& existing_value, std::string* new_value, |
||||
bool* value_changed) const override { |
||||
fprintf(stderr, "Filter(%s)\n", key.ToString().c_str()); |
||||
++count_; |
||||
assert(*value_changed == false); |
||||
return false; |
||||
} |
||||
|
||||
bool FilterMergeOperand(int level, const rocksdb::Slice& key, |
||||
const rocksdb::Slice& existing_value) const override { |
||||
fprintf(stderr, "FilterMerge(%s)\n", key.ToString().c_str()); |
||||
++merge_count_; |
||||
return existing_value == "bad"; |
||||
} |
||||
|
||||
const char* Name() const override { return "MyFilter"; } |
||||
|
||||
mutable int count_ = 0; |
||||
mutable int merge_count_ = 0; |
||||
}; |
||||
|
||||
int main() { |
||||
rocksdb::DB* raw_db; |
||||
rocksdb::Status status; |
||||
|
||||
MyFilter filter; |
||||
|
||||
system("rm -rf /tmp/rocksmergetest"); |
||||
rocksdb::Options options; |
||||
options.create_if_missing = true; |
||||
options.merge_operator.reset(new MyMerge); |
||||
options.compaction_filter = &filter; |
||||
status = rocksdb::DB::Open(options, "/tmp/rocksmergetest", &raw_db); |
||||
assert(status.ok()); |
||||
std::unique_ptr<rocksdb::DB> db(raw_db); |
||||
|
||||
rocksdb::WriteOptions wopts; |
||||
db->Merge(wopts, "0", "bad"); // This is filtered out
|
||||
db->Merge(wopts, "1", "data1"); |
||||
db->Merge(wopts, "1", "bad"); |
||||
db->Merge(wopts, "1", "data2"); |
||||
db->Merge(wopts, "1", "bad"); |
||||
db->Merge(wopts, "3", "data3"); |
||||
db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr); |
||||
fprintf(stderr, "filter.count_ = %d\n", filter.count_); |
||||
assert(filter.count_ == 1); |
||||
fprintf(stderr, "filter.merge_count_ = %d\n", filter.merge_count_); |
||||
assert(filter.merge_count_ == 5); |
||||
} |
||||
@ -0,0 +1,45 @@ |
||||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#pragma once |
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
#include <string> |
||||
|
||||
#include "rocksdb/db.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
struct DumpOptions { |
||||
// Database that will be dumped
|
||||
std::string db_path; |
||||
// File location that will contain dump output
|
||||
std::string dump_location; |
||||
// Dont include db information header in the dump
|
||||
bool anonymous = false; |
||||
}; |
||||
|
||||
class DbDumpTool { |
||||
public: |
||||
bool Run(const DumpOptions& dump_options, |
||||
rocksdb::Options options = rocksdb::Options()); |
||||
}; |
||||
|
||||
struct UndumpOptions { |
||||
// Database that we will load the dumped file into
|
||||
std::string db_path; |
||||
// File location of the dumped file that will be loaded
|
||||
std::string dump_location; |
||||
// Compact the db after loading the dumped file
|
||||
bool compact_db = false; |
||||
}; |
||||
|
||||
class DbUndumpTool { |
||||
public: |
||||
bool Run(const UndumpOptions& undump_options, |
||||
rocksdb::Options options = rocksdb::Options()); |
||||
}; |
||||
} // namespace rocksdb
|
||||
#endif // ROCKSDB_LITE
|
||||
@ -0,0 +1,261 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
#ifndef __STDC_FORMAT_MACROS |
||||
#define __STDC_FORMAT_MACROS |
||||
#endif |
||||
|
||||
#include <inttypes.h> |
||||
#include <iostream> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/db_dump_tool.h" |
||||
#include "rocksdb/env.h" |
||||
#include "util/coding.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
bool DbDumpTool::Run(const DumpOptions& dump_options, |
||||
rocksdb::Options options) { |
||||
rocksdb::DB* dbptr; |
||||
rocksdb::Status status; |
||||
std::unique_ptr<rocksdb::WritableFile> dumpfile; |
||||
char hostname[1024]; |
||||
int64_t timesec; |
||||
std::string abspath; |
||||
char json[4096]; |
||||
|
||||
static const char* magicstr = "ROCKDUMP"; |
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
||||
|
||||
rocksdb::Env* env = rocksdb::Env::Default(); |
||||
|
||||
// Open the database
|
||||
options.create_if_missing = false; |
||||
status = rocksdb::DB::OpenForReadOnly(options, dump_options.db_path, &dbptr); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open database '" << dump_options.db_path |
||||
<< "' for reading: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr); |
||||
|
||||
status = env->NewWritableFile(dump_options.dump_location, &dumpfile, |
||||
rocksdb::EnvOptions()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open dump file '" << dump_options.dump_location |
||||
<< "' for writing: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
rocksdb::Slice magicslice(magicstr, 8); |
||||
status = dumpfile->Append(magicslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
rocksdb::Slice versionslice(versionstr, 8); |
||||
status = dumpfile->Append(versionslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
if (dump_options.anonymous) { |
||||
snprintf(json, sizeof(json), "{}"); |
||||
} else { |
||||
status = env->GetHostName(hostname, sizeof(hostname)); |
||||
status = env->GetCurrentTime(×ec); |
||||
status = env->GetAbsolutePath(dump_options.db_path, &abspath); |
||||
snprintf(json, sizeof(json), |
||||
"{ \"database-path\": \"%s\", \"hostname\": \"%s\", " |
||||
"\"creation-time\": %" PRIi64 " }", |
||||
abspath.c_str(), hostname, timesec); |
||||
} |
||||
|
||||
rocksdb::Slice infoslice(json, strlen(json)); |
||||
char infosize[4]; |
||||
rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size()); |
||||
rocksdb::Slice infosizeslice(infosize, 4); |
||||
status = dumpfile->Append(infosizeslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
status = dumpfile->Append(infoslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
const std::unique_ptr<rocksdb::Iterator> it( |
||||
db->NewIterator(rocksdb::ReadOptions())); |
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
||||
char keysize[4]; |
||||
rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size()); |
||||
rocksdb::Slice keysizeslice(keysize, 4); |
||||
status = dumpfile->Append(keysizeslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
status = dumpfile->Append(it->key()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
char valsize[4]; |
||||
rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size()); |
||||
rocksdb::Slice valsizeslice(valsize, 4); |
||||
status = dumpfile->Append(valsizeslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
status = dumpfile->Append(it->value()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
} |
||||
if (!it->status().ok()) { |
||||
std::cerr << "Database iteration failed: " << status.ToString() |
||||
<< std::endl; |
||||
return false; |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
bool DbUndumpTool::Run(const UndumpOptions& undump_options, |
||||
rocksdb::Options options) { |
||||
rocksdb::DB* dbptr; |
||||
rocksdb::Status status; |
||||
rocksdb::Env* env; |
||||
std::unique_ptr<rocksdb::SequentialFile> dumpfile; |
||||
rocksdb::Slice slice; |
||||
char scratch8[8]; |
||||
|
||||
static const char* magicstr = "ROCKDUMP"; |
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
||||
|
||||
env = rocksdb::Env::Default(); |
||||
|
||||
status = env->NewSequentialFile(undump_options.dump_location, &dumpfile, |
||||
rocksdb::EnvOptions()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open dump file '" << undump_options.dump_location |
||||
<< "' for reading: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 8 || |
||||
memcmp(slice.data(), magicstr, 8) != 0) { |
||||
std::cerr << "File '" << undump_options.dump_location |
||||
<< "' is not a recognizable dump file." << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 8 || |
||||
memcmp(slice.data(), versionstr, 8) != 0) { |
||||
std::cerr << "File '" << undump_options.dump_location |
||||
<< "' version not recognized." << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 4) { |
||||
std::cerr << "Unable to read info blob size." << std::endl; |
||||
return false; |
||||
} |
||||
uint32_t infosize = rocksdb::DecodeFixed32(slice.data()); |
||||
status = dumpfile->Skip(infosize); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
options.create_if_missing = true; |
||||
status = rocksdb::DB::Open(options, undump_options.db_path, &dbptr); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open database '" << undump_options.db_path |
||||
<< "' for writing: " << status.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr); |
||||
|
||||
uint32_t last_keysize = 64; |
||||
size_t last_valsize = 1 << 20; |
||||
std::unique_ptr<char[]> keyscratch(new char[last_keysize]); |
||||
std::unique_ptr<char[]> valscratch(new char[last_valsize]); |
||||
|
||||
while (1) { |
||||
uint32_t keysize, valsize; |
||||
rocksdb::Slice keyslice; |
||||
rocksdb::Slice valslice; |
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 4) break; |
||||
keysize = rocksdb::DecodeFixed32(slice.data()); |
||||
if (keysize > last_keysize) { |
||||
while (keysize > last_keysize) last_keysize *= 2; |
||||
keyscratch = std::unique_ptr<char[]>(new char[last_keysize]); |
||||
} |
||||
|
||||
status = dumpfile->Read(keysize, &keyslice, keyscratch.get()); |
||||
if (!status.ok() || keyslice.size() != keysize) { |
||||
std::cerr << "Key read failure: " |
||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||
<< std::endl; |
||||
return false; |
||||
} |
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 4) { |
||||
std::cerr << "Unable to read value size: " |
||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||
<< std::endl; |
||||
return false; |
||||
} |
||||
valsize = rocksdb::DecodeFixed32(slice.data()); |
||||
if (valsize > last_valsize) { |
||||
while (valsize > last_valsize) last_valsize *= 2; |
||||
valscratch = std::unique_ptr<char[]>(new char[last_valsize]); |
||||
} |
||||
|
||||
status = dumpfile->Read(valsize, &valslice, valscratch.get()); |
||||
if (!status.ok() || valslice.size() != valsize) { |
||||
std::cerr << "Unable to read value: " |
||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||
<< std::endl; |
||||
return false; |
||||
} |
||||
|
||||
status = db->Put(rocksdb::WriteOptions(), keyslice, valslice); |
||||
if (!status.ok()) { |
||||
fprintf(stderr, "Unable to write database entry\n"); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
if (undump_options.compact_db) { |
||||
status = db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr); |
||||
if (!status.ok()) { |
||||
fprintf(stderr, |
||||
"Unable to compact the database after loading the dumped file\n"); |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
} |
||||
} // namespace rocksdb
|
||||
#endif // ROCKSDB_LITE
|
||||
@ -1,154 +1,63 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifndef GFLAGS |
||||
#if !(defined GFLAGS) || defined(ROCKSDB_LITE) |
||||
|
||||
#include <cstdio> |
||||
int main() { |
||||
#ifndef GFLAGS |
||||
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
||||
#endif |
||||
#ifdef ROCKSDB_LITE |
||||
fprintf(stderr, "DbDumpTool is not supported in ROCKSDB_LITE\n"); |
||||
#endif |
||||
return 1; |
||||
} |
||||
#else |
||||
|
||||
#ifndef __STDC_FORMAT_MACROS |
||||
#define __STDC_FORMAT_MACROS |
||||
#endif |
||||
#else |
||||
|
||||
#include <inttypes.h> |
||||
#include <gflags/gflags.h> |
||||
#include <iostream> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "util/coding.h" |
||||
#include "rocksdb/convenience.h" |
||||
#include "rocksdb/db_dump_tool.h" |
||||
|
||||
DEFINE_bool(anonymous, false, "Output an empty information blob."); |
||||
|
||||
void usage(const char* name) { |
||||
std::cout << "usage: " << name << " [--anonymous] <db> <dumpfile>" |
||||
<< std::endl; |
||||
} |
||||
DEFINE_string(db_path, "", "Path to the db that will be dumped"); |
||||
DEFINE_string(dump_location, "", "Path to where the dump file location"); |
||||
DEFINE_bool(anonymous, false, |
||||
"Remove information like db path, creation time from dumped file"); |
||||
DEFINE_string(db_options, "", |
||||
"Options string used to open the database that will be dumped"); |
||||
|
||||
int main(int argc, char** argv) { |
||||
rocksdb::DB* dbptr; |
||||
rocksdb::Options options; |
||||
rocksdb::Status status; |
||||
std::unique_ptr<rocksdb::WritableFile> dumpfile; |
||||
char hostname[1024]; |
||||
int64_t timesec; |
||||
std::string abspath; |
||||
char json[4096]; |
||||
|
||||
GFLAGS::ParseCommandLineFlags(&argc, &argv, true); |
||||
|
||||
static const char* magicstr = "ROCKDUMP"; |
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
||||
|
||||
if (argc != 3) { |
||||
usage(argv[0]); |
||||
exit(1); |
||||
} |
||||
|
||||
rocksdb::Env* env = rocksdb::Env::Default(); |
||||
|
||||
// Open the database
|
||||
options.create_if_missing = false; |
||||
status = rocksdb::DB::OpenForReadOnly(options, argv[1], &dbptr); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open database '" << argv[1] |
||||
<< "' for reading: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr); |
||||
|
||||
status = env->NewWritableFile(argv[2], &dumpfile, rocksdb::EnvOptions()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open dump file '" << argv[2] |
||||
<< "' for writing: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
if (FLAGS_db_path == "" || FLAGS_dump_location == "") { |
||||
fprintf(stderr, "Please set --db_path and --dump_location\n"); |
||||
return 1; |
||||
} |
||||
|
||||
rocksdb::Slice magicslice(magicstr, 8); |
||||
status = dumpfile->Append(magicslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
rocksdb::Slice versionslice(versionstr, 8); |
||||
status = dumpfile->Append(versionslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
rocksdb::DumpOptions dump_options; |
||||
dump_options.db_path = FLAGS_db_path; |
||||
dump_options.dump_location = FLAGS_dump_location; |
||||
dump_options.anonymous = FLAGS_anonymous; |
||||
|
||||
if (FLAGS_anonymous) { |
||||
snprintf(json, sizeof(json), "{}"); |
||||
} else { |
||||
status = env->GetHostName(hostname, sizeof(hostname)); |
||||
status = env->GetCurrentTime(×ec); |
||||
status = env->GetAbsolutePath(argv[1], &abspath); |
||||
snprintf(json, sizeof(json), |
||||
"{ \"database-path\": \"%s\", \"hostname\": \"%s\", " |
||||
"\"creation-time\": %" PRIi64 " }", |
||||
abspath.c_str(), hostname, timesec); |
||||
} |
||||
|
||||
rocksdb::Slice infoslice(json, strlen(json)); |
||||
char infosize[4]; |
||||
rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size()); |
||||
rocksdb::Slice infosizeslice(infosize, 4); |
||||
status = dumpfile->Append(infosizeslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
status = dumpfile->Append(infoslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
const std::unique_ptr<rocksdb::Iterator> it( |
||||
db->NewIterator(rocksdb::ReadOptions())); |
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
||||
char keysize[4]; |
||||
rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size()); |
||||
rocksdb::Slice keysizeslice(keysize, 4); |
||||
status = dumpfile->Append(keysizeslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
status = dumpfile->Append(it->key()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
char valsize[4]; |
||||
rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size()); |
||||
rocksdb::Slice valsizeslice(valsize, 4); |
||||
status = dumpfile->Append(valsizeslice); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
rocksdb::Options db_options; |
||||
if (FLAGS_db_options != "") { |
||||
rocksdb::Options parsed_options; |
||||
rocksdb::Status s = rocksdb::GetOptionsFromString( |
||||
db_options, FLAGS_db_options, &parsed_options); |
||||
if (!s.ok()) { |
||||
fprintf(stderr, "Cannot parse provided db_options\n"); |
||||
return 1; |
||||
} |
||||
status = dumpfile->Append(it->value()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
} |
||||
if (!it->status().ok()) { |
||||
std::cerr << "Database iteration failed: " << status.ToString() |
||||
<< std::endl; |
||||
exit(1); |
||||
db_options = parsed_options; |
||||
} |
||||
|
||||
rocksdb::DbDumpTool tool; |
||||
if (!tool.Run(dump_options, db_options)) { |
||||
return 1; |
||||
} |
||||
return 0; |
||||
} |
||||
|
||||
#endif // GFLAGS
|
||||
#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||
|
||||
@ -1,136 +1,62 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <cstring> |
||||
#include <iostream> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "util/coding.h" |
||||
|
||||
void usage(const char *name) { |
||||
std::cout << "usage: " << name << " <dumpfile> <rocksdb>" << std::endl; |
||||
#if !(defined GFLAGS) || defined(ROCKSDB_LITE) |
||||
|
||||
#include <cstdio> |
||||
int main() { |
||||
#ifndef GFLAGS |
||||
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
||||
#endif |
||||
#ifdef ROCKSDB_LITE |
||||
fprintf(stderr, "DbUndumpTool is not supported in ROCKSDB_LITE\n"); |
||||
#endif |
||||
return 1; |
||||
} |
||||
|
||||
int main(int argc, char **argv) { |
||||
rocksdb::DB *dbptr; |
||||
rocksdb::Options options; |
||||
rocksdb::Status status; |
||||
rocksdb::Env *env; |
||||
std::unique_ptr<rocksdb::SequentialFile> dumpfile; |
||||
rocksdb::Slice slice; |
||||
char scratch8[8]; |
||||
|
||||
static const char *magicstr = "ROCKDUMP"; |
||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
||||
|
||||
if (argc != 3) { |
||||
usage(argv[0]); |
||||
exit(1); |
||||
} |
||||
|
||||
env = rocksdb::Env::Default(); |
||||
|
||||
status = env->NewSequentialFile(argv[1], &dumpfile, rocksdb::EnvOptions()); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open dump file '" << argv[1] |
||||
<< "' for reading: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 8 || |
||||
memcmp(slice.data(), magicstr, 8) != 0) { |
||||
std::cerr << "File '" << argv[1] << "' is not a recognizable dump file." |
||||
<< std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
status = dumpfile->Read(8, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 8 || |
||||
memcmp(slice.data(), versionstr, 8) != 0) { |
||||
std::cerr << "File '" << argv[1] << "' version not recognized." |
||||
<< std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 4) { |
||||
std::cerr << "Unable to read info blob size." << std::endl; |
||||
exit(1); |
||||
} |
||||
uint32_t infosize = rocksdb::DecodeFixed32(slice.data()); |
||||
status = dumpfile->Skip(infosize); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
options.create_if_missing = true; |
||||
status = rocksdb::DB::Open(options, argv[2], &dbptr); |
||||
if (!status.ok()) { |
||||
std::cerr << "Unable to open database '" << argv[2] |
||||
<< "' for writing: " << status.ToString() << std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
const std::unique_ptr<rocksdb::DB> db(dbptr); |
||||
#else |
||||
|
||||
uint32_t last_keysize = 64; |
||||
size_t last_valsize = 1 << 20; |
||||
std::unique_ptr<char[]> keyscratch(new char[last_keysize]); |
||||
std::unique_ptr<char[]> valscratch(new char[last_valsize]); |
||||
#include <gflags/gflags.h> |
||||
#include "rocksdb/convenience.h" |
||||
#include "rocksdb/db_dump_tool.h" |
||||
|
||||
while (1) { |
||||
uint32_t keysize, valsize; |
||||
rocksdb::Slice keyslice; |
||||
rocksdb::Slice valslice; |
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 4) break; |
||||
keysize = rocksdb::DecodeFixed32(slice.data()); |
||||
if (keysize > last_keysize) { |
||||
while (keysize > last_keysize) last_keysize *= 2; |
||||
keyscratch = std::unique_ptr<char[]>(new char[last_keysize]); |
||||
} |
||||
|
||||
status = dumpfile->Read(keysize, &keyslice, keyscratch.get()); |
||||
if (!status.ok() || keyslice.size() != keysize) { |
||||
std::cerr << "Key read failure: " |
||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||
<< std::endl; |
||||
exit(1); |
||||
} |
||||
DEFINE_string(dump_location, "", "Path to the dump file that will be loaded"); |
||||
DEFINE_string(db_path, "", "Path to the db that we will undump the file into"); |
||||
DEFINE_bool(compact, false, "Compact the db after loading the dumped file"); |
||||
DEFINE_string(db_options, "", |
||||
"Options string used to open the database that will be loaded"); |
||||
|
||||
status = dumpfile->Read(4, &slice, scratch8); |
||||
if (!status.ok() || slice.size() != 4) { |
||||
std::cerr << "Unable to read value size: " |
||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||
<< std::endl; |
||||
exit(1); |
||||
} |
||||
valsize = rocksdb::DecodeFixed32(slice.data()); |
||||
if (valsize > last_valsize) { |
||||
while (valsize > last_valsize) last_valsize *= 2; |
||||
valscratch = std::unique_ptr<char[]>(new char[last_valsize]); |
||||
} |
||||
|
||||
status = dumpfile->Read(valsize, &valslice, valscratch.get()); |
||||
if (!status.ok() || valslice.size() != valsize) { |
||||
std::cerr << "Unable to read value: " |
||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||
<< std::endl; |
||||
exit(1); |
||||
} |
||||
|
||||
status = db->Put(rocksdb::WriteOptions(), keyslice, valslice); |
||||
if (!status.ok()) { |
||||
fprintf(stderr, "Unable to write database entry\n"); |
||||
exit(1); |
||||
int main(int argc, char **argv) { |
||||
GFLAGS::ParseCommandLineFlags(&argc, &argv, true); |
||||
|
||||
if (FLAGS_db_path == "" || FLAGS_dump_location == "") { |
||||
fprintf(stderr, "Please set --db_path and --dump_location\n"); |
||||
return 1; |
||||
} |
||||
|
||||
rocksdb::UndumpOptions undump_options; |
||||
undump_options.db_path = FLAGS_db_path; |
||||
undump_options.dump_location = FLAGS_dump_location; |
||||
undump_options.compact_db = FLAGS_compact; |
||||
|
||||
rocksdb::Options db_options; |
||||
if (FLAGS_db_options != "") { |
||||
rocksdb::Options parsed_options; |
||||
rocksdb::Status s = rocksdb::GetOptionsFromString( |
||||
db_options, FLAGS_db_options, &parsed_options); |
||||
if (!s.ok()) { |
||||
fprintf(stderr, "Cannot parse provided db_options\n"); |
||||
return 1; |
||||
} |
||||
db_options = parsed_options; |
||||
} |
||||
|
||||
rocksdb::DbUndumpTool tool; |
||||
if (!tool.Run(undump_options, db_options)) { |
||||
return 1; |
||||
} |
||||
return 0; |
||||
} |
||||
#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue