Praveen Rao
9 years ago
commit
eb24178553
@ -1,6 +1,7 @@ |
|||||||
column_families_example |
|
||||||
simple_example |
|
||||||
c_simple_example |
c_simple_example |
||||||
|
column_families_example |
||||||
compact_files_example |
compact_files_example |
||||||
transaction_example |
compaction_filter_example |
||||||
optimistic_transaction_example |
optimistic_transaction_example |
||||||
|
simple_example |
||||||
|
transaction_example |
||||||
|
|||||||
@ -0,0 +1,84 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include <rocksdb/compaction_filter.h> |
||||||
|
#include <rocksdb/db.h> |
||||||
|
#include <rocksdb/merge_operator.h> |
||||||
|
#include <rocksdb/options.h> |
||||||
|
|
||||||
|
class MyMerge : public rocksdb::MergeOperator { |
||||||
|
public: |
||||||
|
bool FullMerge(const rocksdb::Slice& key, |
||||||
|
const rocksdb::Slice* existing_value, |
||||||
|
const std::deque<std::string>& operand_list, |
||||||
|
std::string* new_value, |
||||||
|
rocksdb::Logger* logger) const override { |
||||||
|
new_value->clear(); |
||||||
|
if (existing_value != nullptr) { |
||||||
|
new_value->assign(existing_value->data(), existing_value->size()); |
||||||
|
} |
||||||
|
for (const std::string& m : operand_list) { |
||||||
|
fprintf(stderr, "Merge(%s)\n", m.c_str()); |
||||||
|
assert(m != "bad"); // the compaction filter filters out bad values
|
||||||
|
new_value->assign(m); |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
const char* Name() const override { return "MyMerge"; } |
||||||
|
}; |
||||||
|
|
||||||
|
class MyFilter : public rocksdb::CompactionFilter { |
||||||
|
public: |
||||||
|
bool Filter(int level, const rocksdb::Slice& key, |
||||||
|
const rocksdb::Slice& existing_value, std::string* new_value, |
||||||
|
bool* value_changed) const override { |
||||||
|
fprintf(stderr, "Filter(%s)\n", key.ToString().c_str()); |
||||||
|
++count_; |
||||||
|
assert(*value_changed == false); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
bool FilterMergeOperand(int level, const rocksdb::Slice& key, |
||||||
|
const rocksdb::Slice& existing_value) const override { |
||||||
|
fprintf(stderr, "FilterMerge(%s)\n", key.ToString().c_str()); |
||||||
|
++merge_count_; |
||||||
|
return existing_value == "bad"; |
||||||
|
} |
||||||
|
|
||||||
|
const char* Name() const override { return "MyFilter"; } |
||||||
|
|
||||||
|
mutable int count_ = 0; |
||||||
|
mutable int merge_count_ = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
int main() { |
||||||
|
rocksdb::DB* raw_db; |
||||||
|
rocksdb::Status status; |
||||||
|
|
||||||
|
MyFilter filter; |
||||||
|
|
||||||
|
system("rm -rf /tmp/rocksmergetest"); |
||||||
|
rocksdb::Options options; |
||||||
|
options.create_if_missing = true; |
||||||
|
options.merge_operator.reset(new MyMerge); |
||||||
|
options.compaction_filter = &filter; |
||||||
|
status = rocksdb::DB::Open(options, "/tmp/rocksmergetest", &raw_db); |
||||||
|
assert(status.ok()); |
||||||
|
std::unique_ptr<rocksdb::DB> db(raw_db); |
||||||
|
|
||||||
|
rocksdb::WriteOptions wopts; |
||||||
|
db->Merge(wopts, "0", "bad"); // This is filtered out
|
||||||
|
db->Merge(wopts, "1", "data1"); |
||||||
|
db->Merge(wopts, "1", "bad"); |
||||||
|
db->Merge(wopts, "1", "data2"); |
||||||
|
db->Merge(wopts, "1", "bad"); |
||||||
|
db->Merge(wopts, "3", "data3"); |
||||||
|
db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr); |
||||||
|
fprintf(stderr, "filter.count_ = %d\n", filter.count_); |
||||||
|
assert(filter.count_ == 1); |
||||||
|
fprintf(stderr, "filter.merge_count_ = %d\n", filter.merge_count_); |
||||||
|
assert(filter.merge_count_ == 5); |
||||||
|
} |
||||||
@ -0,0 +1,45 @@ |
|||||||
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "rocksdb/db.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
struct DumpOptions { |
||||||
|
// Database that will be dumped
|
||||||
|
std::string db_path; |
||||||
|
// File location that will contain dump output
|
||||||
|
std::string dump_location; |
||||||
|
// Dont include db information header in the dump
|
||||||
|
bool anonymous = false; |
||||||
|
}; |
||||||
|
|
||||||
|
class DbDumpTool { |
||||||
|
public: |
||||||
|
bool Run(const DumpOptions& dump_options, |
||||||
|
rocksdb::Options options = rocksdb::Options()); |
||||||
|
}; |
||||||
|
|
||||||
|
struct UndumpOptions { |
||||||
|
// Database that we will load the dumped file into
|
||||||
|
std::string db_path; |
||||||
|
// File location of the dumped file that will be loaded
|
||||||
|
std::string dump_location; |
||||||
|
// Compact the db after loading the dumped file
|
||||||
|
bool compact_db = false; |
||||||
|
}; |
||||||
|
|
||||||
|
class DbUndumpTool { |
||||||
|
public: |
||||||
|
bool Run(const UndumpOptions& undump_options, |
||||||
|
rocksdb::Options options = rocksdb::Options()); |
||||||
|
}; |
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,261 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#ifndef __STDC_FORMAT_MACROS |
||||||
|
#define __STDC_FORMAT_MACROS |
||||||
|
#endif |
||||||
|
|
||||||
|
#include <inttypes.h> |
||||||
|
#include <iostream> |
||||||
|
|
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/db_dump_tool.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
bool DbDumpTool::Run(const DumpOptions& dump_options, |
||||||
|
rocksdb::Options options) { |
||||||
|
rocksdb::DB* dbptr; |
||||||
|
rocksdb::Status status; |
||||||
|
std::unique_ptr<rocksdb::WritableFile> dumpfile; |
||||||
|
char hostname[1024]; |
||||||
|
int64_t timesec; |
||||||
|
std::string abspath; |
||||||
|
char json[4096]; |
||||||
|
|
||||||
|
static const char* magicstr = "ROCKDUMP"; |
||||||
|
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
||||||
|
|
||||||
|
rocksdb::Env* env = rocksdb::Env::Default(); |
||||||
|
|
||||||
|
// Open the database
|
||||||
|
options.create_if_missing = false; |
||||||
|
status = rocksdb::DB::OpenForReadOnly(options, dump_options.db_path, &dbptr); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Unable to open database '" << dump_options.db_path |
||||||
|
<< "' for reading: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
const std::unique_ptr<rocksdb::DB> db(dbptr); |
||||||
|
|
||||||
|
status = env->NewWritableFile(dump_options.dump_location, &dumpfile, |
||||||
|
rocksdb::EnvOptions()); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Unable to open dump file '" << dump_options.dump_location |
||||||
|
<< "' for writing: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
rocksdb::Slice magicslice(magicstr, 8); |
||||||
|
status = dumpfile->Append(magicslice); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
rocksdb::Slice versionslice(versionstr, 8); |
||||||
|
status = dumpfile->Append(versionslice); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (dump_options.anonymous) { |
||||||
|
snprintf(json, sizeof(json), "{}"); |
||||||
|
} else { |
||||||
|
status = env->GetHostName(hostname, sizeof(hostname)); |
||||||
|
status = env->GetCurrentTime(×ec); |
||||||
|
status = env->GetAbsolutePath(dump_options.db_path, &abspath); |
||||||
|
snprintf(json, sizeof(json), |
||||||
|
"{ \"database-path\": \"%s\", \"hostname\": \"%s\", " |
||||||
|
"\"creation-time\": %" PRIi64 " }", |
||||||
|
abspath.c_str(), hostname, timesec); |
||||||
|
} |
||||||
|
|
||||||
|
rocksdb::Slice infoslice(json, strlen(json)); |
||||||
|
char infosize[4]; |
||||||
|
rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size()); |
||||||
|
rocksdb::Slice infosizeslice(infosize, 4); |
||||||
|
status = dumpfile->Append(infosizeslice); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
status = dumpfile->Append(infoslice); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
const std::unique_ptr<rocksdb::Iterator> it( |
||||||
|
db->NewIterator(rocksdb::ReadOptions())); |
||||||
|
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
||||||
|
char keysize[4]; |
||||||
|
rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size()); |
||||||
|
rocksdb::Slice keysizeslice(keysize, 4); |
||||||
|
status = dumpfile->Append(keysizeslice); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
status = dumpfile->Append(it->key()); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
char valsize[4]; |
||||||
|
rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size()); |
||||||
|
rocksdb::Slice valsizeslice(valsize, 4); |
||||||
|
status = dumpfile->Append(valsizeslice); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
status = dumpfile->Append(it->value()); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Append failed: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
if (!it->status().ok()) { |
||||||
|
std::cerr << "Database iteration failed: " << status.ToString() |
||||||
|
<< std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
bool DbUndumpTool::Run(const UndumpOptions& undump_options, |
||||||
|
rocksdb::Options options) { |
||||||
|
rocksdb::DB* dbptr; |
||||||
|
rocksdb::Status status; |
||||||
|
rocksdb::Env* env; |
||||||
|
std::unique_ptr<rocksdb::SequentialFile> dumpfile; |
||||||
|
rocksdb::Slice slice; |
||||||
|
char scratch8[8]; |
||||||
|
|
||||||
|
static const char* magicstr = "ROCKDUMP"; |
||||||
|
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
||||||
|
|
||||||
|
env = rocksdb::Env::Default(); |
||||||
|
|
||||||
|
status = env->NewSequentialFile(undump_options.dump_location, &dumpfile, |
||||||
|
rocksdb::EnvOptions()); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Unable to open dump file '" << undump_options.dump_location |
||||||
|
<< "' for reading: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
status = dumpfile->Read(8, &slice, scratch8); |
||||||
|
if (!status.ok() || slice.size() != 8 || |
||||||
|
memcmp(slice.data(), magicstr, 8) != 0) { |
||||||
|
std::cerr << "File '" << undump_options.dump_location |
||||||
|
<< "' is not a recognizable dump file." << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
status = dumpfile->Read(8, &slice, scratch8); |
||||||
|
if (!status.ok() || slice.size() != 8 || |
||||||
|
memcmp(slice.data(), versionstr, 8) != 0) { |
||||||
|
std::cerr << "File '" << undump_options.dump_location |
||||||
|
<< "' version not recognized." << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
status = dumpfile->Read(4, &slice, scratch8); |
||||||
|
if (!status.ok() || slice.size() != 4) { |
||||||
|
std::cerr << "Unable to read info blob size." << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
uint32_t infosize = rocksdb::DecodeFixed32(slice.data()); |
||||||
|
status = dumpfile->Skip(infosize); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
options.create_if_missing = true; |
||||||
|
status = rocksdb::DB::Open(options, undump_options.db_path, &dbptr); |
||||||
|
if (!status.ok()) { |
||||||
|
std::cerr << "Unable to open database '" << undump_options.db_path |
||||||
|
<< "' for writing: " << status.ToString() << std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
const std::unique_ptr<rocksdb::DB> db(dbptr); |
||||||
|
|
||||||
|
uint32_t last_keysize = 64; |
||||||
|
size_t last_valsize = 1 << 20; |
||||||
|
std::unique_ptr<char[]> keyscratch(new char[last_keysize]); |
||||||
|
std::unique_ptr<char[]> valscratch(new char[last_valsize]); |
||||||
|
|
||||||
|
while (1) { |
||||||
|
uint32_t keysize, valsize; |
||||||
|
rocksdb::Slice keyslice; |
||||||
|
rocksdb::Slice valslice; |
||||||
|
|
||||||
|
status = dumpfile->Read(4, &slice, scratch8); |
||||||
|
if (!status.ok() || slice.size() != 4) break; |
||||||
|
keysize = rocksdb::DecodeFixed32(slice.data()); |
||||||
|
if (keysize > last_keysize) { |
||||||
|
while (keysize > last_keysize) last_keysize *= 2; |
||||||
|
keyscratch = std::unique_ptr<char[]>(new char[last_keysize]); |
||||||
|
} |
||||||
|
|
||||||
|
status = dumpfile->Read(keysize, &keyslice, keyscratch.get()); |
||||||
|
if (!status.ok() || keyslice.size() != keysize) { |
||||||
|
std::cerr << "Key read failure: " |
||||||
|
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||||
|
<< std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
status = dumpfile->Read(4, &slice, scratch8); |
||||||
|
if (!status.ok() || slice.size() != 4) { |
||||||
|
std::cerr << "Unable to read value size: " |
||||||
|
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||||
|
<< std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
valsize = rocksdb::DecodeFixed32(slice.data()); |
||||||
|
if (valsize > last_valsize) { |
||||||
|
while (valsize > last_valsize) last_valsize *= 2; |
||||||
|
valscratch = std::unique_ptr<char[]>(new char[last_valsize]); |
||||||
|
} |
||||||
|
|
||||||
|
status = dumpfile->Read(valsize, &valslice, valscratch.get()); |
||||||
|
if (!status.ok() || valslice.size() != valsize) { |
||||||
|
std::cerr << "Unable to read value: " |
||||||
|
<< (status.ok() ? "insufficient data" : status.ToString()) |
||||||
|
<< std::endl; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
status = db->Put(rocksdb::WriteOptions(), keyslice, valslice); |
||||||
|
if (!status.ok()) { |
||||||
|
fprintf(stderr, "Unable to write database entry\n"); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (undump_options.compact_db) { |
||||||
|
status = db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr); |
||||||
|
if (!status.ok()) { |
||||||
|
fprintf(stderr, |
||||||
|
"Unable to compact the database after loading the dumped file\n"); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -1,154 +1,63 @@ |
|||||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
// This source code is licensed under the BSD-style license found in the
|
// This source code is licensed under the BSD-style license found in the
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
#ifndef GFLAGS |
#if !(defined GFLAGS) || defined(ROCKSDB_LITE) |
||||||
|
|
||||||
#include <cstdio> |
#include <cstdio> |
||||||
int main() { |
int main() { |
||||||
|
#ifndef GFLAGS |
||||||
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
||||||
|
#endif |
||||||
|
#ifdef ROCKSDB_LITE |
||||||
|
fprintf(stderr, "DbDumpTool is not supported in ROCKSDB_LITE\n"); |
||||||
|
#endif |
||||||
return 1; |
return 1; |
||||||
} |
} |
||||||
#else |
|
||||||
|
|
||||||
#ifndef __STDC_FORMAT_MACROS |
#else |
||||||
#define __STDC_FORMAT_MACROS |
|
||||||
#endif |
|
||||||
|
|
||||||
#include <inttypes.h> |
|
||||||
#include <gflags/gflags.h> |
#include <gflags/gflags.h> |
||||||
#include <iostream> |
#include "rocksdb/convenience.h" |
||||||
|
#include "rocksdb/db_dump_tool.h" |
||||||
#include "rocksdb/db.h" |
|
||||||
#include "rocksdb/env.h" |
|
||||||
#include "util/coding.h" |
|
||||||
|
|
||||||
DEFINE_bool(anonymous, false, "Output an empty information blob."); |
DEFINE_string(db_path, "", "Path to the db that will be dumped"); |
||||||
|
DEFINE_string(dump_location, "", "Path to where the dump file location"); |
||||||
void usage(const char* name) { |
DEFINE_bool(anonymous, false, |
||||||
std::cout << "usage: " << name << " [--anonymous] <db> <dumpfile>" |
"Remove information like db path, creation time from dumped file"); |
||||||
<< std::endl; |
DEFINE_string(db_options, "", |
||||||
} |
"Options string used to open the database that will be dumped"); |
||||||
|
|
||||||
int main(int argc, char** argv) { |
int main(int argc, char** argv) { |
||||||
rocksdb::DB* dbptr; |
|
||||||
rocksdb::Options options; |
|
||||||
rocksdb::Status status; |
|
||||||
std::unique_ptr<rocksdb::WritableFile> dumpfile; |
|
||||||
char hostname[1024]; |
|
||||||
int64_t timesec; |
|
||||||
std::string abspath; |
|
||||||
char json[4096]; |
|
||||||
|
|
||||||
GFLAGS::ParseCommandLineFlags(&argc, &argv, true); |
GFLAGS::ParseCommandLineFlags(&argc, &argv, true); |
||||||
|
|
||||||
static const char* magicstr = "ROCKDUMP"; |
if (FLAGS_db_path == "" || FLAGS_dump_location == "") { |
||||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
fprintf(stderr, "Please set --db_path and --dump_location\n"); |
||||||
|
return 1; |
||||||
if (argc != 3) { |
|
||||||
usage(argv[0]); |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
rocksdb::Env* env = rocksdb::Env::Default(); |
|
||||||
|
|
||||||
// Open the database
|
|
||||||
options.create_if_missing = false; |
|
||||||
status = rocksdb::DB::OpenForReadOnly(options, argv[1], &dbptr); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Unable to open database '" << argv[1] |
|
||||||
<< "' for reading: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
const std::unique_ptr<rocksdb::DB> db(dbptr); |
|
||||||
|
|
||||||
status = env->NewWritableFile(argv[2], &dumpfile, rocksdb::EnvOptions()); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Unable to open dump file '" << argv[2] |
|
||||||
<< "' for writing: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
rocksdb::Slice magicslice(magicstr, 8); |
|
||||||
status = dumpfile->Append(magicslice); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
rocksdb::Slice versionslice(versionstr, 8); |
|
||||||
status = dumpfile->Append(versionslice); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
} |
||||||
|
|
||||||
if (FLAGS_anonymous) { |
rocksdb::DumpOptions dump_options; |
||||||
snprintf(json, sizeof(json), "{}"); |
dump_options.db_path = FLAGS_db_path; |
||||||
} else { |
dump_options.dump_location = FLAGS_dump_location; |
||||||
status = env->GetHostName(hostname, sizeof(hostname)); |
dump_options.anonymous = FLAGS_anonymous; |
||||||
status = env->GetCurrentTime(×ec); |
|
||||||
status = env->GetAbsolutePath(argv[1], &abspath); |
|
||||||
snprintf(json, sizeof(json), |
|
||||||
"{ \"database-path\": \"%s\", \"hostname\": \"%s\", " |
|
||||||
"\"creation-time\": %" PRIi64 " }", |
|
||||||
abspath.c_str(), hostname, timesec); |
|
||||||
} |
|
||||||
|
|
||||||
rocksdb::Slice infoslice(json, strlen(json)); |
rocksdb::Options db_options; |
||||||
char infosize[4]; |
if (FLAGS_db_options != "") { |
||||||
rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size()); |
rocksdb::Options parsed_options; |
||||||
rocksdb::Slice infosizeslice(infosize, 4); |
rocksdb::Status s = rocksdb::GetOptionsFromString( |
||||||
status = dumpfile->Append(infosizeslice); |
db_options, FLAGS_db_options, &parsed_options); |
||||||
if (!status.ok()) { |
if (!s.ok()) { |
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
fprintf(stderr, "Cannot parse provided db_options\n"); |
||||||
exit(1); |
return 1; |
||||||
} |
|
||||||
status = dumpfile->Append(infoslice); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
const std::unique_ptr<rocksdb::Iterator> it( |
|
||||||
db->NewIterator(rocksdb::ReadOptions())); |
|
||||||
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
|
||||||
char keysize[4]; |
|
||||||
rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size()); |
|
||||||
rocksdb::Slice keysizeslice(keysize, 4); |
|
||||||
status = dumpfile->Append(keysizeslice); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
} |
||||||
status = dumpfile->Append(it->key()); |
db_options = parsed_options; |
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
} |
||||||
|
|
||||||
char valsize[4]; |
rocksdb::DbDumpTool tool; |
||||||
rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size()); |
if (!tool.Run(dump_options, db_options)) { |
||||||
rocksdb::Slice valsizeslice(valsize, 4); |
return 1; |
||||||
status = dumpfile->Append(valsizeslice); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
status = dumpfile->Append(it->value()); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Append failed: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
} |
|
||||||
if (!it->status().ok()) { |
|
||||||
std::cerr << "Database iteration failed: " << status.ToString() |
|
||||||
<< std::endl; |
|
||||||
exit(1); |
|
||||||
} |
} |
||||||
|
|
||||||
return 0; |
return 0; |
||||||
} |
} |
||||||
|
#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||||
#endif // GFLAGS
|
|
||||||
|
|||||||
@ -1,136 +1,62 @@ |
|||||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
// This source code is licensed under the BSD-style license found in the
|
// This source code is licensed under the BSD-style license found in the
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
#include <cstring> |
#if !(defined GFLAGS) || defined(ROCKSDB_LITE) |
||||||
#include <iostream> |
|
||||||
|
#include <cstdio> |
||||||
#include "rocksdb/db.h" |
int main() { |
||||||
#include "rocksdb/env.h" |
#ifndef GFLAGS |
||||||
#include "util/coding.h" |
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
||||||
|
#endif |
||||||
void usage(const char *name) { |
#ifdef ROCKSDB_LITE |
||||||
std::cout << "usage: " << name << " <dumpfile> <rocksdb>" << std::endl; |
fprintf(stderr, "DbUndumpTool is not supported in ROCKSDB_LITE\n"); |
||||||
|
#endif |
||||||
|
return 1; |
||||||
} |
} |
||||||
|
|
||||||
int main(int argc, char **argv) { |
#else |
||||||
rocksdb::DB *dbptr; |
|
||||||
rocksdb::Options options; |
|
||||||
rocksdb::Status status; |
|
||||||
rocksdb::Env *env; |
|
||||||
std::unique_ptr<rocksdb::SequentialFile> dumpfile; |
|
||||||
rocksdb::Slice slice; |
|
||||||
char scratch8[8]; |
|
||||||
|
|
||||||
static const char *magicstr = "ROCKDUMP"; |
|
||||||
static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; |
|
||||||
|
|
||||||
if (argc != 3) { |
|
||||||
usage(argv[0]); |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
env = rocksdb::Env::Default(); |
|
||||||
|
|
||||||
status = env->NewSequentialFile(argv[1], &dumpfile, rocksdb::EnvOptions()); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Unable to open dump file '" << argv[1] |
|
||||||
<< "' for reading: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
status = dumpfile->Read(8, &slice, scratch8); |
|
||||||
if (!status.ok() || slice.size() != 8 || |
|
||||||
memcmp(slice.data(), magicstr, 8) != 0) { |
|
||||||
std::cerr << "File '" << argv[1] << "' is not a recognizable dump file." |
|
||||||
<< std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
status = dumpfile->Read(8, &slice, scratch8); |
|
||||||
if (!status.ok() || slice.size() != 8 || |
|
||||||
memcmp(slice.data(), versionstr, 8) != 0) { |
|
||||||
std::cerr << "File '" << argv[1] << "' version not recognized." |
|
||||||
<< std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
status = dumpfile->Read(4, &slice, scratch8); |
|
||||||
if (!status.ok() || slice.size() != 4) { |
|
||||||
std::cerr << "Unable to read info blob size." << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
uint32_t infosize = rocksdb::DecodeFixed32(slice.data()); |
|
||||||
status = dumpfile->Skip(infosize); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
options.create_if_missing = true; |
|
||||||
status = rocksdb::DB::Open(options, argv[2], &dbptr); |
|
||||||
if (!status.ok()) { |
|
||||||
std::cerr << "Unable to open database '" << argv[2] |
|
||||||
<< "' for writing: " << status.ToString() << std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
const std::unique_ptr<rocksdb::DB> db(dbptr); |
#include <gflags/gflags.h> |
||||||
|
#include "rocksdb/convenience.h" |
||||||
|
#include "rocksdb/db_dump_tool.h" |
||||||
|
|
||||||
uint32_t last_keysize = 64; |
DEFINE_string(dump_location, "", "Path to the dump file that will be loaded"); |
||||||
size_t last_valsize = 1 << 20; |
DEFINE_string(db_path, "", "Path to the db that we will undump the file into"); |
||||||
std::unique_ptr<char[]> keyscratch(new char[last_keysize]); |
DEFINE_bool(compact, false, "Compact the db after loading the dumped file"); |
||||||
std::unique_ptr<char[]> valscratch(new char[last_valsize]); |
DEFINE_string(db_options, "", |
||||||
|
"Options string used to open the database that will be loaded"); |
||||||
|
|
||||||
while (1) { |
int main(int argc, char **argv) { |
||||||
uint32_t keysize, valsize; |
GFLAGS::ParseCommandLineFlags(&argc, &argv, true); |
||||||
rocksdb::Slice keyslice; |
|
||||||
rocksdb::Slice valslice; |
|
||||||
|
|
||||||
status = dumpfile->Read(4, &slice, scratch8); |
if (FLAGS_db_path == "" || FLAGS_dump_location == "") { |
||||||
if (!status.ok() || slice.size() != 4) break; |
fprintf(stderr, "Please set --db_path and --dump_location\n"); |
||||||
keysize = rocksdb::DecodeFixed32(slice.data()); |
return 1; |
||||||
if (keysize > last_keysize) { |
|
||||||
while (keysize > last_keysize) last_keysize *= 2; |
|
||||||
keyscratch = std::unique_ptr<char[]>(new char[last_keysize]); |
|
||||||
} |
} |
||||||
|
|
||||||
status = dumpfile->Read(keysize, &keyslice, keyscratch.get()); |
rocksdb::UndumpOptions undump_options; |
||||||
if (!status.ok() || keyslice.size() != keysize) { |
undump_options.db_path = FLAGS_db_path; |
||||||
std::cerr << "Key read failure: " |
undump_options.dump_location = FLAGS_dump_location; |
||||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
undump_options.compact_db = FLAGS_compact; |
||||||
<< std::endl; |
|
||||||
exit(1); |
|
||||||
} |
|
||||||
|
|
||||||
status = dumpfile->Read(4, &slice, scratch8); |
rocksdb::Options db_options; |
||||||
if (!status.ok() || slice.size() != 4) { |
if (FLAGS_db_options != "") { |
||||||
std::cerr << "Unable to read value size: " |
rocksdb::Options parsed_options; |
||||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
rocksdb::Status s = rocksdb::GetOptionsFromString( |
||||||
<< std::endl; |
db_options, FLAGS_db_options, &parsed_options); |
||||||
exit(1); |
if (!s.ok()) { |
||||||
} |
fprintf(stderr, "Cannot parse provided db_options\n"); |
||||||
valsize = rocksdb::DecodeFixed32(slice.data()); |
return 1; |
||||||
if (valsize > last_valsize) { |
|
||||||
while (valsize > last_valsize) last_valsize *= 2; |
|
||||||
valscratch = std::unique_ptr<char[]>(new char[last_valsize]); |
|
||||||
} |
} |
||||||
|
db_options = parsed_options; |
||||||
status = dumpfile->Read(valsize, &valslice, valscratch.get()); |
|
||||||
if (!status.ok() || valslice.size() != valsize) { |
|
||||||
std::cerr << "Unable to read value: " |
|
||||||
<< (status.ok() ? "insufficient data" : status.ToString()) |
|
||||||
<< std::endl; |
|
||||||
exit(1); |
|
||||||
} |
} |
||||||
|
|
||||||
status = db->Put(rocksdb::WriteOptions(), keyslice, valslice); |
rocksdb::DbUndumpTool tool; |
||||||
if (!status.ok()) { |
if (!tool.Run(undump_options, db_options)) { |
||||||
fprintf(stderr, "Unable to write database entry\n"); |
return 1; |
||||||
exit(1); |
|
||||||
} |
|
||||||
} |
} |
||||||
|
|
||||||
return 0; |
return 0; |
||||||
} |
} |
||||||
|
#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue