NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tools: use provided options instead of the default (#4839)

Browse Source 
Summary:
The current implementation hardcode the default options in different
places, which makes it impossible to support other environments (like
encrypted environment).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4839

Differential Revision: D13573578

Pulled By: sagar0

fbshipit-source-id: 76b58b4b758902798d10ff2f52d9f39abff015e7
main
Huachao Huang 6 years ago committed by Facebook Github Bot
parent e4feb78606
commit 74f7d7551e
6 changed files with 135 additions and 65 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      include/rocksdb/sst_dump_tool.h
  2. 54
      tools/ldb_cmd.cc
  3. 36
      tools/ldb_cmd_test.cc
  4. 91
      tools/sst_dump_test.cc
  5. 11
      tools/sst_dump_tool.cc
  6. 4
      tools/sst_dump_tool_imp.h

4
include/rocksdb/sst_dump_tool.h
Unescape View File

@ -5,11 +5,13 @@
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#pragma once #pragma once
#include "rocksdb/options.h"
namespace rocksdb { namespace rocksdb {
class SSTDumpTool { class SSTDumpTool {
public: public:
int Run(int argc, char** argv); int Run(int argc, char** argv, Options options = Options());
}; };
} // namespace rocksdb } // namespace rocksdb

54
tools/ldb_cmd.cc
Unescape View File

@ -81,10 +81,12 @@ const char* LDBCommand::DELIM = " ==> ";
namespace { namespace {
void DumpWalFile(std::string wal_file, bool print_header, bool print_values, void DumpWalFile(Options options, std::string wal_file, bool print_header,
bool is_write_committed, LDBCommandExecuteResult* exec_state); bool print_values, bool is_write_committed,
LDBCommandExecuteResult* exec_state);
void DumpSstFile(std::string filename, bool output_hex, bool show_properties); void DumpSstFile(Options options, std::string filename, bool output_hex,
bool show_properties);
}; };
LDBCommand* LDBCommand::InitFromCmdLineArgs( LDBCommand* LDBCommand::InitFromCmdLineArgs(
@ -928,8 +930,8 @@ void DBLoaderCommand::DoCommand() {
namespace { namespace {
void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) { void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
Options options; bool json) {
EnvOptions sopt; EnvOptions sopt;
std::string dbname("dummy"); std::string dbname("dummy");
std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10, std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
@ -1030,7 +1032,7 @@ void ManifestDumpCommand::DoCommand() {
printf("Processing Manifest file %s\n", manifestfile.c_str()); printf("Processing Manifest file %s\n", manifestfile.c_str());
} }
DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_); DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
if (verbose_) { if (verbose_) {
printf("Processing Manifest file %s done\n", manifestfile.c_str()); printf("Processing Manifest file %s done\n", manifestfile.c_str());
@ -1425,14 +1427,15 @@ void DBDumperCommand::DoCommand() {
switch (type) { switch (type) {
case kLogFile: case kLogFile:
// TODO(myabandeh): allow configuring is_write_commited // TODO(myabandeh): allow configuring is_write_commited
DumpWalFile(path_, /* print_header_ */ true, /* print_values_ */ true, DumpWalFile(options_, path_, /* print_header_ */ true,
true /* is_write_commited */, &exec_state_); /* print_values_ */ true, true /* is_write_commited */,
&exec_state_);
break; break;
case kTableFile: case kTableFile:
DumpSstFile(path_, is_key_hex_, /* show_properties */ true); DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true);
break; break;
case kDescriptorFile: case kDescriptorFile:
DumpManifestFile(path_, /* verbose_ */ false, is_key_hex_, DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
/* json_ */ false); /* json_ */ false);
break; break;
default: default:
@ -1960,16 +1963,17 @@ class InMemoryHandler : public WriteBatch::Handler {
bool write_after_commit_; bool write_after_commit_;
}; };
void DumpWalFile(std::string wal_file, bool print_header, bool print_values, void DumpWalFile(Options options, std::string wal_file, bool print_header,
bool is_write_committed, LDBCommandExecuteResult* exec_state) { bool print_values, bool is_write_committed,
Env* env_ = Env::Default(); LDBCommandExecuteResult* exec_state) {
EnvOptions soptions; Env* env = options.env;
EnvOptions soptions(options);
std::unique_ptr<SequentialFileReader> wal_file_reader; std::unique_ptr<SequentialFileReader> wal_file_reader;
Status status; Status status;
{ {
std::unique_ptr<SequentialFile> file; std::unique_ptr<SequentialFile> file;
status = env_->NewSequentialFile(wal_file, &file, soptions); status = env->NewSequentialFile(wal_file, &file, soptions);
if (status.ok()) { if (status.ok()) {
wal_file_reader.reset( wal_file_reader.reset(
new SequentialFileReader(std::move(file), wal_file)); new SequentialFileReader(std::move(file), wal_file));
@ -1997,9 +2001,8 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
// bogus input, carry on as best we can // bogus input, carry on as best we can
log_number = 0; log_number = 0;
} }
DBOptions db_options; log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter,
log::Reader reader(db_options.info_log, std::move(wal_file_reader), true /* checksum */, log_number,
&reporter, true /* checksum */, log_number,
false /* retry_after_eof */); false /* retry_after_eof */);
std::string scratch; std::string scratch;
WriteBatch batch; WriteBatch batch;
@ -2079,8 +2082,8 @@ void WALDumperCommand::Help(std::string& ret) {
} }
void WALDumperCommand::DoCommand() { void WALDumperCommand::DoCommand() {
DumpWalFile(wal_file_, print_header_, print_values_, is_write_committed_, DumpWalFile(options_, wal_file_, print_header_, print_values_,
&exec_state_); is_write_committed_, &exec_state_);
} }
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
@ -2836,7 +2839,8 @@ void RestoreCommand::DoCommand() {
namespace { namespace {
void DumpSstFile(std::string filename, bool output_hex, bool show_properties) { void DumpSstFile(Options options, std::string filename, bool output_hex,
bool show_properties) {
std::string from_key; std::string from_key;
std::string to_key; std::string to_key;
if (filename.length() <= 4 || if (filename.length() <= 4 ||
@ -2845,7 +2849,7 @@ void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
return; return;
} }
// no verification // no verification
rocksdb::SstFileDumper dumper(filename, false, output_hex); rocksdb::SstFileDumper dumper(options, filename, false, output_hex);
Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(), Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
false, // has_from false, // has_from
from_key, false, // has_to from_key, false, // has_to
@ -2911,7 +2915,7 @@ void DBFileDumperCommand::DoCommand() {
manifest_filename.resize(manifest_filename.size() - 1); manifest_filename.resize(manifest_filename.size() - 1);
std::string manifest_filepath = db_->GetName() + "/" + manifest_filename; std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
std::cout << manifest_filepath << std::endl; std::cout << manifest_filepath << std::endl;
DumpManifestFile(manifest_filepath, false, false, false); DumpManifestFile(options_, manifest_filepath, false, false, false);
std::cout << std::endl; std::cout << std::endl;
std::cout << "SST Files" << std::endl; std::cout << "SST Files" << std::endl;
@ -2922,7 +2926,7 @@ void DBFileDumperCommand::DoCommand() {
std::string filename = fileMetadata.db_path + fileMetadata.name; std::string filename = fileMetadata.db_path + fileMetadata.name;
std::cout << filename << " level:" << fileMetadata.level << std::endl; std::cout << filename << " level:" << fileMetadata.level << std::endl;
std::cout << "------------------------------" << std::endl; std::cout << "------------------------------" << std::endl;
DumpSstFile(filename, false, true); DumpSstFile(options_, filename, false, true);
std::cout << std::endl; std::cout << std::endl;
} }
std::cout << std::endl; std::cout << std::endl;
@ -2939,7 +2943,7 @@ void DBFileDumperCommand::DoCommand() {
std::string filename = db_->GetOptions().wal_dir + wal->PathName(); std::string filename = db_->GetOptions().wal_dir + wal->PathName();
std::cout << filename << std::endl; std::cout << filename << std::endl;
// TODO(myabandeh): allow configuring is_write_commited // TODO(myabandeh): allow configuring is_write_commited
DumpWalFile(filename, true, true, true /* is_write_commited */, DumpWalFile(options_, filename, true, true, true /* is_write_commited */,
&exec_state_); &exec_state_);
} }
} }

36
tools/ldb_cmd_test.cc
Unescape View File

@ -12,6 +12,8 @@ using std::string;
using std::vector; using std::vector;
using std::map; using std::map;
namespace rocksdb {
class LdbCmdTest : public testing::Test {}; class LdbCmdTest : public testing::Test {};
TEST_F(LdbCmdTest, HexToString) { TEST_F(LdbCmdTest, HexToString) {
@ -47,6 +49,40 @@ TEST_F(LdbCmdTest, HexToStringBadInputs) {
} }
} }
TEST_F(LdbCmdTest, MemEnv) {
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
Options opts;
opts.env = env.get();
opts.create_if_missing = true;
DB* db = nullptr;
std::string dbname = test::TmpDir();
ASSERT_OK(DB::Open(opts, dbname, &db));
WriteOptions wopts;
for (int i = 0; i < 100; i++) {
char buf[16];
snprintf(buf, sizeof(buf), "%08d", i);
ASSERT_OK(db->Put(wopts, buf, buf));
}
FlushOptions fopts;
fopts.wait = true;
ASSERT_OK(db->Flush(fopts));
delete db;
char arg1[] = "./ldb";
char arg2[1024];
snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
char arg3[] = "dump_live_files";
char* argv[] = {arg1, arg2, arg3};
rocksdb::LDBTool tool;
tool.Run(3, argv, opts);
}
} // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();

91
tools/sst_dump_test.cc
Unescape View File

@ -38,24 +38,18 @@ static std::string MakeValue(int i) {
return key.Encode().ToString(); return key.Encode().ToString();
} }
void createSST(const std::string& file_name, void createSST(const Options& opts, const std::string& file_name) {
const BlockBasedTableOptions& table_options) { Env* env = opts.env;
std::shared_ptr<rocksdb::TableFactory> tf; EnvOptions env_options(opts);
tf.reset(new rocksdb::BlockBasedTableFactory(table_options));
std::unique_ptr<WritableFile> file;
Env* env = Env::Default();
EnvOptions env_options;
ReadOptions read_options; ReadOptions read_options;
Options opts;
const ImmutableCFOptions imoptions(opts); const ImmutableCFOptions imoptions(opts);
const MutableCFOptions moptions(opts); const MutableCFOptions moptions(opts);
rocksdb::InternalKeyComparator ikc(opts.comparator); rocksdb::InternalKeyComparator ikc(opts.comparator);
std::unique_ptr<TableBuilder> tb; std::unique_ptr<TableBuilder> tb;
std::unique_ptr<WritableFile> file;
ASSERT_OK(env->NewWritableFile(file_name, &file, env_options)); ASSERT_OK(env->NewWritableFile(file_name, &file, env_options));
opts.table_factory = tf;
std::vector<std::unique_ptr<IntTblPropCollectorFactory> > std::vector<std::unique_ptr<IntTblPropCollectorFactory> >
int_tbl_prop_collector_factories; int_tbl_prop_collector_factories;
std::unique_ptr<WritableFileWriter> file_writer( std::unique_ptr<WritableFileWriter> file_writer(
@ -80,8 +74,8 @@ void createSST(const std::string& file_name,
file_writer->Close(); file_writer->Close();
} }
void cleanup(const std::string& file_name) { void cleanup(const Options& opts, const std::string& file_name) {
Env* env = Env::Default(); Env* env = opts.env;
env->DeleteFile(file_name); env->DeleteFile(file_name);
std::string outfile_name = file_name.substr(0, file_name.length() - 4); std::string outfile_name = file_name.substr(0, file_name.length() - 4);
outfile_name.append("_dump.txt"); outfile_name.append("_dump.txt");
@ -94,8 +88,6 @@ class SSTDumpToolTest : public testing::Test {
std::string testDir_; std::string testDir_;
public: public:
BlockBasedTableOptions table_options_;
SSTDumpToolTest() { testDir_ = test::TmpDir(); } SSTDumpToolTest() { testDir_ = test::TmpDir(); }
~SSTDumpToolTest() {} ~SSTDumpToolTest() {}
@ -119,88 +111,121 @@ class SSTDumpToolTest : public testing::Test {
}; };
TEST_F(SSTDumpToolTest, EmptyFilter) { TEST_F(SSTDumpToolTest, EmptyFilter) {
Options opts;
std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_); createSST(opts, file_path);
char* usage[3]; char* usage[3];
PopulateCommandArgs(file_path, "--command=raw", usage); PopulateCommandArgs(file_path, "--command=raw", usage);
rocksdb::SSTDumpTool tool; rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage)); ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path); cleanup(opts, file_path);
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
delete[] usage[i]; delete[] usage[i];
} }
} }
TEST_F(SSTDumpToolTest, FilterBlock) { TEST_F(SSTDumpToolTest, FilterBlock) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true)); Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_); createSST(opts, file_path);
char* usage[3]; char* usage[3];
PopulateCommandArgs(file_path, "--command=raw", usage); PopulateCommandArgs(file_path, "--command=raw", usage);
rocksdb::SSTDumpTool tool; rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage)); ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path); cleanup(opts, file_path);
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
delete[] usage[i]; delete[] usage[i];
} }
} }
TEST_F(SSTDumpToolTest, FullFilterBlock) { TEST_F(SSTDumpToolTest, FullFilterBlock) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_); createSST(opts, file_path);
char* usage[3]; char* usage[3];
PopulateCommandArgs(file_path, "--command=raw", usage); PopulateCommandArgs(file_path, "--command=raw", usage);
rocksdb::SSTDumpTool tool; rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage)); ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path); cleanup(opts, file_path);
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
delete[] usage[i]; delete[] usage[i];
} }
} }
TEST_F(SSTDumpToolTest, GetProperties) { TEST_F(SSTDumpToolTest, GetProperties) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_); createSST(opts, file_path);
char* usage[3]; char* usage[3];
PopulateCommandArgs(file_path, "--show_properties", usage); PopulateCommandArgs(file_path, "--show_properties", usage);
rocksdb::SSTDumpTool tool; rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage)); ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path); cleanup(opts, file_path);
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
delete[] usage[i]; delete[] usage[i];
} }
} }
TEST_F(SSTDumpToolTest, CompressedSizes) { TEST_F(SSTDumpToolTest, CompressedSizes) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_); createSST(opts, file_path);
char* usage[3]; char* usage[3];
PopulateCommandArgs(file_path, "--command=recompress", usage); PopulateCommandArgs(file_path, "--command=recompress", usage);
rocksdb::SSTDumpTool tool; rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage)); ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path); cleanup(opts, file_path);
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
delete[] usage[i]; delete[] usage[i];
} }
} }
TEST_F(SSTDumpToolTest, MemEnv) {
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
Options opts;
opts.env = env.get();
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(opts, file_path);
char* usage[3];
PopulateCommandArgs(file_path, "--command=verify_checksum", usage);
rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(opts, file_path);
for (int i = 0; i < 3; i++) {
delete[] usage[i];
}
}
} // namespace rocksdb } // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {

11
tools/sst_dump_tool.cc
Unescape View File

@ -43,12 +43,14 @@
namespace rocksdb { namespace rocksdb {
SstFileDumper::SstFileDumper(const std::string& file_path, bool verify_checksum, SstFileDumper::SstFileDumper(const Options& options,
const std::string& file_path, bool verify_checksum,
bool output_hex) bool output_hex)
: file_name_(file_path), : file_name_(file_path),
read_num_(0), read_num_(0),
verify_checksum_(verify_checksum), verify_checksum_(verify_checksum),
output_hex_(output_hex), output_hex_(output_hex),
options_(options),
ioptions_(options_), ioptions_(options_),
moptions_(ColumnFamilyOptions(options_)), moptions_(ColumnFamilyOptions(options_)),
internal_comparator_(BytewiseComparator()) { internal_comparator_(BytewiseComparator()) {
@ -417,7 +419,7 @@ void print_help() {
} // namespace } // namespace
int SSTDumpTool::Run(int argc, char** argv) { int SSTDumpTool::Run(int argc, char** argv, Options options) {
const char* dir_or_file = nullptr; const char* dir_or_file = nullptr;
uint64_t read_num = std::numeric_limits<uint64_t>::max(); uint64_t read_num = std::numeric_limits<uint64_t>::max();
std::string command; std::string command;
@ -545,7 +547,7 @@ int SSTDumpTool::Run(int argc, char** argv) {
} }
std::vector<std::string> filenames; std::vector<std::string> filenames;
rocksdb::Env* env = rocksdb::Env::Default(); rocksdb::Env* env = options.env;
rocksdb::Status st = env->GetChildren(dir_or_file, &filenames); rocksdb::Status st = env->GetChildren(dir_or_file, &filenames);
bool dir = true; bool dir = true;
if (!st.ok()) { if (!st.ok()) {
@ -570,7 +572,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
filename = std::string(dir_or_file) + "/" + filename; filename = std::string(dir_or_file) + "/" + filename;
} }
rocksdb::SstFileDumper dumper(filename, verify_checksum, output_hex); rocksdb::SstFileDumper dumper(options, filename, verify_checksum,
output_hex);
if (!dumper.getStatus().ok()) { if (!dumper.getStatus().ok()) {
fprintf(stderr, "%s: %s\n", filename.c_str(), fprintf(stderr, "%s: %s\n", filename.c_str(),
dumper.getStatus().ToString().c_str()); dumper.getStatus().ToString().c_str());

4
tools/sst_dump_tool_imp.h
Unescape View File

@ -17,8 +17,8 @@ namespace rocksdb {
class SstFileDumper { class SstFileDumper {
public: public:
explicit SstFileDumper(const std::string& file_name, bool verify_checksum, explicit SstFileDumper(const Options& options, const std::string& file_name,
bool output_hex); bool verify_checksum, bool output_hex);
Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from, Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from,
const std::string& from_key, bool has_to, const std::string& from_key, bool has_to,

Write Preview
Loading…
Cancel
Save