diff --git a/include/rocksdb/db_dump_tool.h b/include/rocksdb/db_dump_tool.h new file mode 100644 index 000000000..67575a94b --- /dev/null +++ b/include/rocksdb/db_dump_tool.h @@ -0,0 +1,45 @@ +// Copyright (c) 2015, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#pragma once +#ifndef ROCKSDB_LITE + +#include + +#include "rocksdb/db.h" + +namespace rocksdb { + +struct DumpOptions { + // Database that will be dumped + std::string db_path; + // File location that will contain dump output + std::string dump_location; + // Dont include db information header in the dump + bool anonymous = false; +}; + +class DbDumpTool { + public: + bool Run(const DumpOptions& dump_options, + rocksdb::Options options = rocksdb::Options()); +}; + +struct UndumpOptions { + // Database that we will load the dumped file into + std::string db_path; + // File location of the dumped file that will be loaded + std::string dump_location; + // Compact the db after loading the dumped file + bool compact_db = false; +}; + +class DbUndumpTool { + public: + bool Run(const UndumpOptions& undump_options, + rocksdb::Options options = rocksdb::Options()); +}; +} // namespace rocksdb +#endif // ROCKSDB_LITE diff --git a/src.mk b/src.mk index 4a5fd335f..5e9ed1644 100644 --- a/src.mk +++ b/src.mk @@ -76,6 +76,7 @@ LIB_SOURCES = \ table/plain_table_reader.cc \ table/table_properties.cc \ table/two_level_iterator.cc \ + tools/dump/db_dump_tool.cc \ util/arena.cc \ util/auto_roll_logger.cc \ util/bloom.cc \ diff --git a/tools/dump/db_dump_tool.cc b/tools/dump/db_dump_tool.cc new file mode 100644 index 000000000..389e65dba --- /dev/null +++ b/tools/dump/db_dump_tool.cc @@ -0,0 +1,261 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef ROCKSDB_LITE + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/db_dump_tool.h" +#include "rocksdb/env.h" +#include "util/coding.h" + +namespace rocksdb { + +bool DbDumpTool::Run(const DumpOptions& dump_options, + rocksdb::Options options) { + rocksdb::DB* dbptr; + rocksdb::Status status; + std::unique_ptr dumpfile; + char hostname[1024]; + int64_t timesec; + std::string abspath; + char json[4096]; + + static const char* magicstr = "ROCKDUMP"; + static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; + + rocksdb::Env* env = rocksdb::Env::Default(); + + // Open the database + options.create_if_missing = false; + status = rocksdb::DB::OpenForReadOnly(options, dump_options.db_path, &dbptr); + if (!status.ok()) { + std::cerr << "Unable to open database '" << dump_options.db_path + << "' for reading: " << status.ToString() << std::endl; + return false; + } + + const std::unique_ptr db(dbptr); + + status = env->NewWritableFile(dump_options.dump_location, &dumpfile, + rocksdb::EnvOptions()); + if (!status.ok()) { + std::cerr << "Unable to open dump file '" << dump_options.dump_location + << "' for writing: " << status.ToString() << std::endl; + return false; + } + + rocksdb::Slice magicslice(magicstr, 8); + status = dumpfile->Append(magicslice); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + + rocksdb::Slice versionslice(versionstr, 8); + status = dumpfile->Append(versionslice); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + + if (dump_options.anonymous) { + snprintf(json, sizeof(json), "{}"); + } else { + status = env->GetHostName(hostname, sizeof(hostname)); + status = env->GetCurrentTime(×ec); + status = env->GetAbsolutePath(dump_options.db_path, &abspath); + snprintf(json, sizeof(json), + "{ \"database-path\": \"%s\", \"hostname\": \"%s\", " + "\"creation-time\": %" PRIi64 " }", + abspath.c_str(), hostname, timesec); + } + + rocksdb::Slice infoslice(json, strlen(json)); + char infosize[4]; + rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size()); + rocksdb::Slice infosizeslice(infosize, 4); + status = dumpfile->Append(infosizeslice); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + status = dumpfile->Append(infoslice); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + + const std::unique_ptr it( + db->NewIterator(rocksdb::ReadOptions())); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + char keysize[4]; + rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size()); + rocksdb::Slice keysizeslice(keysize, 4); + status = dumpfile->Append(keysizeslice); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + status = dumpfile->Append(it->key()); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + + char valsize[4]; + rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size()); + rocksdb::Slice valsizeslice(valsize, 4); + status = dumpfile->Append(valsizeslice); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + status = dumpfile->Append(it->value()); + if (!status.ok()) { + std::cerr << "Append failed: " << status.ToString() << std::endl; + return false; + } + } + if (!it->status().ok()) { + std::cerr << "Database iteration failed: " << status.ToString() + << std::endl; + return false; + } + return true; +} + +bool DbUndumpTool::Run(const UndumpOptions& undump_options, + rocksdb::Options options) { + rocksdb::DB* dbptr; + rocksdb::Status status; + rocksdb::Env* env; + std::unique_ptr dumpfile; + rocksdb::Slice slice; + char scratch8[8]; + + static const char* magicstr = "ROCKDUMP"; + static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; + + env = rocksdb::Env::Default(); + + status = env->NewSequentialFile(undump_options.dump_location, &dumpfile, + rocksdb::EnvOptions()); + if (!status.ok()) { + std::cerr << "Unable to open dump file '" << undump_options.dump_location + << "' for reading: " << status.ToString() << std::endl; + return false; + } + + status = dumpfile->Read(8, &slice, scratch8); + if (!status.ok() || slice.size() != 8 || + memcmp(slice.data(), magicstr, 8) != 0) { + std::cerr << "File '" << undump_options.dump_location + << "' is not a recognizable dump file." << std::endl; + return false; + } + + status = dumpfile->Read(8, &slice, scratch8); + if (!status.ok() || slice.size() != 8 || + memcmp(slice.data(), versionstr, 8) != 0) { + std::cerr << "File '" << undump_options.dump_location + << "' version not recognized." << std::endl; + return false; + } + + status = dumpfile->Read(4, &slice, scratch8); + if (!status.ok() || slice.size() != 4) { + std::cerr << "Unable to read info blob size." << std::endl; + return false; + } + uint32_t infosize = rocksdb::DecodeFixed32(slice.data()); + status = dumpfile->Skip(infosize); + if (!status.ok()) { + std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl; + return false; + } + + options.create_if_missing = true; + status = rocksdb::DB::Open(options, undump_options.db_path, &dbptr); + if (!status.ok()) { + std::cerr << "Unable to open database '" << undump_options.db_path + << "' for writing: " << status.ToString() << std::endl; + return false; + } + + const std::unique_ptr db(dbptr); + + uint32_t last_keysize = 64; + size_t last_valsize = 1 << 20; + std::unique_ptr keyscratch(new char[last_keysize]); + std::unique_ptr valscratch(new char[last_valsize]); + + while (1) { + uint32_t keysize, valsize; + rocksdb::Slice keyslice; + rocksdb::Slice valslice; + + status = dumpfile->Read(4, &slice, scratch8); + if (!status.ok() || slice.size() != 4) break; + keysize = rocksdb::DecodeFixed32(slice.data()); + if (keysize > last_keysize) { + while (keysize > last_keysize) last_keysize *= 2; + keyscratch = std::unique_ptr(new char[last_keysize]); + } + + status = dumpfile->Read(keysize, &keyslice, keyscratch.get()); + if (!status.ok() || keyslice.size() != keysize) { + std::cerr << "Key read failure: " + << (status.ok() ? "insufficient data" : status.ToString()) + << std::endl; + return false; + } + + status = dumpfile->Read(4, &slice, scratch8); + if (!status.ok() || slice.size() != 4) { + std::cerr << "Unable to read value size: " + << (status.ok() ? "insufficient data" : status.ToString()) + << std::endl; + return false; + } + valsize = rocksdb::DecodeFixed32(slice.data()); + if (valsize > last_valsize) { + while (valsize > last_valsize) last_valsize *= 2; + valscratch = std::unique_ptr(new char[last_valsize]); + } + + status = dumpfile->Read(valsize, &valslice, valscratch.get()); + if (!status.ok() || valslice.size() != valsize) { + std::cerr << "Unable to read value: " + << (status.ok() ? "insufficient data" : status.ToString()) + << std::endl; + return false; + } + + status = db->Put(rocksdb::WriteOptions(), keyslice, valslice); + if (!status.ok()) { + fprintf(stderr, "Unable to write database entry\n"); + return false; + } + } + + if (undump_options.compact_db) { + status = db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr); + if (!status.ok()) { + fprintf(stderr, + "Unable to compact the database after loading the dumped file\n"); + return false; + } + } + return true; +} +} // namespace rocksdb +#endif // ROCKSDB_LITE diff --git a/tools/dump/rocksdb_dump.cc b/tools/dump/rocksdb_dump.cc index 95b275b94..2bfc6cee3 100644 --- a/tools/dump/rocksdb_dump.cc +++ b/tools/dump/rocksdb_dump.cc @@ -1,154 +1,63 @@ -// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// Copyright (c) 2015, Facebook, Inc. All rights reserved. // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -#ifndef GFLAGS +#if !(defined GFLAGS) || defined(ROCKSDB_LITE) + #include int main() { +#ifndef GFLAGS fprintf(stderr, "Please install gflags to run rocksdb tools\n"); +#endif +#ifdef ROCKSDB_LITE + fprintf(stderr, "DbDumpTool is not supported in ROCKSDB_LITE\n"); +#endif return 1; } -#else -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif +#else -#include #include -#include - -#include "rocksdb/db.h" -#include "rocksdb/env.h" -#include "util/coding.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db_dump_tool.h" -DEFINE_bool(anonymous, false, "Output an empty information blob."); - -void usage(const char* name) { - std::cout << "usage: " << name << " [--anonymous] " - << std::endl; -} +DEFINE_string(db_path, "", "Path to the db that will be dumped"); +DEFINE_string(dump_location, "", "Path to where the dump file location"); +DEFINE_bool(anonymous, false, + "Remove information like db path, creation time from dumped file"); +DEFINE_string(db_options, "", + "Options string used to open the database that will be dumped"); int main(int argc, char** argv) { - rocksdb::DB* dbptr; - rocksdb::Options options; - rocksdb::Status status; - std::unique_ptr dumpfile; - char hostname[1024]; - int64_t timesec; - std::string abspath; - char json[4096]; - GFLAGS::ParseCommandLineFlags(&argc, &argv, true); - static const char* magicstr = "ROCKDUMP"; - static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; - - if (argc != 3) { - usage(argv[0]); - exit(1); - } - - rocksdb::Env* env = rocksdb::Env::Default(); - - // Open the database - options.create_if_missing = false; - status = rocksdb::DB::OpenForReadOnly(options, argv[1], &dbptr); - if (!status.ok()) { - std::cerr << "Unable to open database '" << argv[1] - << "' for reading: " << status.ToString() << std::endl; - exit(1); - } - - const std::unique_ptr db(dbptr); - - status = env->NewWritableFile(argv[2], &dumpfile, rocksdb::EnvOptions()); - if (!status.ok()) { - std::cerr << "Unable to open dump file '" << argv[2] - << "' for writing: " << status.ToString() << std::endl; - exit(1); + if (FLAGS_db_path == "" || FLAGS_dump_location == "") { + fprintf(stderr, "Please set --db_path and --dump_location\n"); + return 1; } - rocksdb::Slice magicslice(magicstr, 8); - status = dumpfile->Append(magicslice); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); - } - - rocksdb::Slice versionslice(versionstr, 8); - status = dumpfile->Append(versionslice); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); - } + rocksdb::DumpOptions dump_options; + dump_options.db_path = FLAGS_db_path; + dump_options.dump_location = FLAGS_dump_location; + dump_options.anonymous = FLAGS_anonymous; - if (FLAGS_anonymous) { - snprintf(json, sizeof(json), "{}"); - } else { - status = env->GetHostName(hostname, sizeof(hostname)); - status = env->GetCurrentTime(×ec); - status = env->GetAbsolutePath(argv[1], &abspath); - snprintf(json, sizeof(json), - "{ \"database-path\": \"%s\", \"hostname\": \"%s\", " - "\"creation-time\": %" PRIi64 " }", - abspath.c_str(), hostname, timesec); - } - - rocksdb::Slice infoslice(json, strlen(json)); - char infosize[4]; - rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size()); - rocksdb::Slice infosizeslice(infosize, 4); - status = dumpfile->Append(infosizeslice); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); - } - status = dumpfile->Append(infoslice); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); - } - - const std::unique_ptr it( - db->NewIterator(rocksdb::ReadOptions())); - for (it->SeekToFirst(); it->Valid(); it->Next()) { - char keysize[4]; - rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size()); - rocksdb::Slice keysizeslice(keysize, 4); - status = dumpfile->Append(keysizeslice); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); - } - status = dumpfile->Append(it->key()); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); - } - - char valsize[4]; - rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size()); - rocksdb::Slice valsizeslice(valsize, 4); - status = dumpfile->Append(valsizeslice); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); + rocksdb::Options db_options; + if (FLAGS_db_options != "") { + rocksdb::Options parsed_options; + rocksdb::Status s = rocksdb::GetOptionsFromString( + db_options, FLAGS_db_options, &parsed_options); + if (!s.ok()) { + fprintf(stderr, "Cannot parse provided db_options\n"); + return 1; } - status = dumpfile->Append(it->value()); - if (!status.ok()) { - std::cerr << "Append failed: " << status.ToString() << std::endl; - exit(1); - } - } - if (!it->status().ok()) { - std::cerr << "Database iteration failed: " << status.ToString() - << std::endl; - exit(1); + db_options = parsed_options; } + rocksdb::DbDumpTool tool; + if (!tool.Run(dump_options, db_options)) { + return 1; + } return 0; } - -#endif // GFLAGS +#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE) diff --git a/tools/dump/rocksdb_undump.cc b/tools/dump/rocksdb_undump.cc index c356c78d6..81034f0ce 100644 --- a/tools/dump/rocksdb_undump.cc +++ b/tools/dump/rocksdb_undump.cc @@ -1,136 +1,62 @@ -// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// Copyright (c) 2015, Facebook, Inc. All rights reserved. // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -#include -#include - -#include "rocksdb/db.h" -#include "rocksdb/env.h" -#include "util/coding.h" - -void usage(const char *name) { - std::cout << "usage: " << name << " " << std::endl; +#if !(defined GFLAGS) || defined(ROCKSDB_LITE) + +#include +int main() { +#ifndef GFLAGS + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); +#endif +#ifdef ROCKSDB_LITE + fprintf(stderr, "DbUndumpTool is not supported in ROCKSDB_LITE\n"); +#endif + return 1; } -int main(int argc, char **argv) { - rocksdb::DB *dbptr; - rocksdb::Options options; - rocksdb::Status status; - rocksdb::Env *env; - std::unique_ptr dumpfile; - rocksdb::Slice slice; - char scratch8[8]; - - static const char *magicstr = "ROCKDUMP"; - static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1}; - - if (argc != 3) { - usage(argv[0]); - exit(1); - } - - env = rocksdb::Env::Default(); - - status = env->NewSequentialFile(argv[1], &dumpfile, rocksdb::EnvOptions()); - if (!status.ok()) { - std::cerr << "Unable to open dump file '" << argv[1] - << "' for reading: " << status.ToString() << std::endl; - exit(1); - } - - status = dumpfile->Read(8, &slice, scratch8); - if (!status.ok() || slice.size() != 8 || - memcmp(slice.data(), magicstr, 8) != 0) { - std::cerr << "File '" << argv[1] << "' is not a recognizable dump file." - << std::endl; - exit(1); - } - - status = dumpfile->Read(8, &slice, scratch8); - if (!status.ok() || slice.size() != 8 || - memcmp(slice.data(), versionstr, 8) != 0) { - std::cerr << "File '" << argv[1] << "' version not recognized." - << std::endl; - exit(1); - } - - status = dumpfile->Read(4, &slice, scratch8); - if (!status.ok() || slice.size() != 4) { - std::cerr << "Unable to read info blob size." << std::endl; - exit(1); - } - uint32_t infosize = rocksdb::DecodeFixed32(slice.data()); - status = dumpfile->Skip(infosize); - if (!status.ok()) { - std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl; - exit(1); - } - - options.create_if_missing = true; - status = rocksdb::DB::Open(options, argv[2], &dbptr); - if (!status.ok()) { - std::cerr << "Unable to open database '" << argv[2] - << "' for writing: " << status.ToString() << std::endl; - exit(1); - } - - const std::unique_ptr db(dbptr); +#else - uint32_t last_keysize = 64; - size_t last_valsize = 1 << 20; - std::unique_ptr keyscratch(new char[last_keysize]); - std::unique_ptr valscratch(new char[last_valsize]); +#include +#include "rocksdb/convenience.h" +#include "rocksdb/db_dump_tool.h" - while (1) { - uint32_t keysize, valsize; - rocksdb::Slice keyslice; - rocksdb::Slice valslice; - - status = dumpfile->Read(4, &slice, scratch8); - if (!status.ok() || slice.size() != 4) break; - keysize = rocksdb::DecodeFixed32(slice.data()); - if (keysize > last_keysize) { - while (keysize > last_keysize) last_keysize *= 2; - keyscratch = std::unique_ptr(new char[last_keysize]); - } - - status = dumpfile->Read(keysize, &keyslice, keyscratch.get()); - if (!status.ok() || keyslice.size() != keysize) { - std::cerr << "Key read failure: " - << (status.ok() ? "insufficient data" : status.ToString()) - << std::endl; - exit(1); - } +DEFINE_string(dump_location, "", "Path to the dump file that will be loaded"); +DEFINE_string(db_path, "", "Path to the db that we will undump the file into"); +DEFINE_bool(compact, false, "Compact the db after loading the dumped file"); +DEFINE_string(db_options, "", + "Options string used to open the database that will be loaded"); - status = dumpfile->Read(4, &slice, scratch8); - if (!status.ok() || slice.size() != 4) { - std::cerr << "Unable to read value size: " - << (status.ok() ? "insufficient data" : status.ToString()) - << std::endl; - exit(1); - } - valsize = rocksdb::DecodeFixed32(slice.data()); - if (valsize > last_valsize) { - while (valsize > last_valsize) last_valsize *= 2; - valscratch = std::unique_ptr(new char[last_valsize]); - } - - status = dumpfile->Read(valsize, &valslice, valscratch.get()); - if (!status.ok() || valslice.size() != valsize) { - std::cerr << "Unable to read value: " - << (status.ok() ? "insufficient data" : status.ToString()) - << std::endl; - exit(1); - } - - status = db->Put(rocksdb::WriteOptions(), keyslice, valslice); - if (!status.ok()) { - fprintf(stderr, "Unable to write database entry\n"); - exit(1); +int main(int argc, char **argv) { + GFLAGS::ParseCommandLineFlags(&argc, &argv, true); + + if (FLAGS_db_path == "" || FLAGS_dump_location == "") { + fprintf(stderr, "Please set --db_path and --dump_location\n"); + return 1; + } + + rocksdb::UndumpOptions undump_options; + undump_options.db_path = FLAGS_db_path; + undump_options.dump_location = FLAGS_dump_location; + undump_options.compact_db = FLAGS_compact; + + rocksdb::Options db_options; + if (FLAGS_db_options != "") { + rocksdb::Options parsed_options; + rocksdb::Status s = rocksdb::GetOptionsFromString( + db_options, FLAGS_db_options, &parsed_options); + if (!s.ok()) { + fprintf(stderr, "Cannot parse provided db_options\n"); + return 1; } + db_options = parsed_options; } + rocksdb::DbUndumpTool tool; + if (!tool.Run(undump_options, db_options)) { + return 1; + } return 0; } +#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE) diff --git a/tools/rocksdb_dump_test.sh b/tools/rocksdb_dump_test.sh index b423979eb..5c8b5c30a 100755 --- a/tools/rocksdb_dump_test.sh +++ b/tools/rocksdb_dump_test.sh @@ -2,6 +2,6 @@ TESTDIR=`mktemp -d /tmp/rocksdb-dump-test.XXXXX` DUMPFILE="tools/sample-dump.dmp" # Verify that the sample dump file is undumpable and then redumpable. -./rocksdb_undump $DUMPFILE $TESTDIR/db -./rocksdb_dump --anonymous $TESTDIR/db $TESTDIR/dump +./rocksdb_undump --dump_location=$DUMPFILE --db_path=$TESTDIR/db +./rocksdb_dump --anonymous --db_path=$TESTDIR/db --dump_location=$TESTDIR/dump cmp $DUMPFILE $TESTDIR/dump