NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tools: use provided options instead of the default (#4839)

Browse Source 
Summary:
The current implementation hardcode the default options in different
places, which makes it impossible to support other environments (like
encrypted environment).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4839

Differential Revision: D13573578

Pulled By: sagar0

fbshipit-source-id: 76b58b4b758902798d10ff2f52d9f39abff015e7
main
Huachao Huang 6 years ago committed by Facebook Github Bot
parent e4feb78606
commit 74f7d7551e
6 changed files with 135 additions and 65 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      include/rocksdb/sst_dump_tool.h
  2. 54
      tools/ldb_cmd.cc
  3. 36
      tools/ldb_cmd_test.cc
  4. 91
      tools/sst_dump_test.cc
  5. 11
      tools/sst_dump_tool.cc
  6. 4
      tools/sst_dump_tool_imp.h

4
include/rocksdb/sst_dump_tool.h
Unescape View File

@ -5,11 +5,13 @@
#ifndef ROCKSDB_LITE
#pragma once
#include "rocksdb/options.h"
namespace rocksdb {
class SSTDumpTool {
public:
int Run(int argc, char** argv);
int Run(int argc, char** argv, Options options = Options());
};
} // namespace rocksdb

54
tools/ldb_cmd.cc
Unescape View File

@ -81,10 +81,12 @@ const char* LDBCommand::DELIM = " ==> ";
namespace {
void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
bool is_write_committed, LDBCommandExecuteResult* exec_state);
void DumpWalFile(Options options, std::string wal_file, bool print_header,
bool print_values, bool is_write_committed,
LDBCommandExecuteResult* exec_state);
void DumpSstFile(std::string filename, bool output_hex, bool show_properties);
void DumpSstFile(Options options, std::string filename, bool output_hex,
bool show_properties);
};
LDBCommand* LDBCommand::InitFromCmdLineArgs(
@ -928,8 +930,8 @@ void DBLoaderCommand::DoCommand() {
namespace {
void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) {
Options options;
void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
bool json) {
EnvOptions sopt;
std::string dbname("dummy");
std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
@ -1030,7 +1032,7 @@ void ManifestDumpCommand::DoCommand() {
printf("Processing Manifest file %s\n", manifestfile.c_str());
}
DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_);
DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
if (verbose_) {
printf("Processing Manifest file %s done\n", manifestfile.c_str());
@ -1425,14 +1427,15 @@ void DBDumperCommand::DoCommand() {
switch (type) {
case kLogFile:
// TODO(myabandeh): allow configuring is_write_commited
DumpWalFile(path_, /* print_header_ */ true, /* print_values_ */ true,
true /* is_write_commited */, &exec_state_);
DumpWalFile(options_, path_, /* print_header_ */ true,
/* print_values_ */ true, true /* is_write_commited */,
&exec_state_);
break;
case kTableFile:
DumpSstFile(path_, is_key_hex_, /* show_properties */ true);
DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true);
break;
case kDescriptorFile:
DumpManifestFile(path_, /* verbose_ */ false, is_key_hex_,
DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
/* json_ */ false);
break;
default:
@ -1960,16 +1963,17 @@ class InMemoryHandler : public WriteBatch::Handler {
bool write_after_commit_;
};
void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
bool is_write_committed, LDBCommandExecuteResult* exec_state) {
Env* env_ = Env::Default();
EnvOptions soptions;
void DumpWalFile(Options options, std::string wal_file, bool print_header,
bool print_values, bool is_write_committed,
LDBCommandExecuteResult* exec_state) {
Env* env = options.env;
EnvOptions soptions(options);
std::unique_ptr<SequentialFileReader> wal_file_reader;
Status status;
{
std::unique_ptr<SequentialFile> file;
status = env_->NewSequentialFile(wal_file, &file, soptions);
status = env->NewSequentialFile(wal_file, &file, soptions);
if (status.ok()) {
wal_file_reader.reset(
new SequentialFileReader(std::move(file), wal_file));
@ -1997,9 +2001,8 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
// bogus input, carry on as best we can
log_number = 0;
}
DBOptions db_options;
log::Reader reader(db_options.info_log, std::move(wal_file_reader),
&reporter, true /* checksum */, log_number,
log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter,
true /* checksum */, log_number,
false /* retry_after_eof */);
std::string scratch;
WriteBatch batch;
@ -2079,8 +2082,8 @@ void WALDumperCommand::Help(std::string& ret) {
}
void WALDumperCommand::DoCommand() {
DumpWalFile(wal_file_, print_header_, print_values_, is_write_committed_,
&exec_state_);
DumpWalFile(options_, wal_file_, print_header_, print_values_,
is_write_committed_, &exec_state_);
}
// ----------------------------------------------------------------------------
@ -2836,7 +2839,8 @@ void RestoreCommand::DoCommand() {
namespace {
void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
void DumpSstFile(Options options, std::string filename, bool output_hex,
bool show_properties) {
std::string from_key;
std::string to_key;
if (filename.length() <= 4 ||
@ -2845,7 +2849,7 @@ void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
return;
}
// no verification
rocksdb::SstFileDumper dumper(filename, false, output_hex);
rocksdb::SstFileDumper dumper(options, filename, false, output_hex);
Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
false, // has_from
from_key, false, // has_to
@ -2911,7 +2915,7 @@ void DBFileDumperCommand::DoCommand() {
manifest_filename.resize(manifest_filename.size() - 1);
std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
std::cout << manifest_filepath << std::endl;
DumpManifestFile(manifest_filepath, false, false, false);
DumpManifestFile(options_, manifest_filepath, false, false, false);
std::cout << std::endl;
std::cout << "SST Files" << std::endl;
@ -2922,7 +2926,7 @@ void DBFileDumperCommand::DoCommand() {
std::string filename = fileMetadata.db_path + fileMetadata.name;
std::cout << filename << " level:" << fileMetadata.level << std::endl;
std::cout << "------------------------------" << std::endl;
DumpSstFile(filename, false, true);
DumpSstFile(options_, filename, false, true);
std::cout << std::endl;
}
std::cout << std::endl;
@ -2939,7 +2943,7 @@ void DBFileDumperCommand::DoCommand() {
std::string filename = db_->GetOptions().wal_dir + wal->PathName();
std::cout << filename << std::endl;
// TODO(myabandeh): allow configuring is_write_commited
DumpWalFile(filename, true, true, true /* is_write_commited */,
DumpWalFile(options_, filename, true, true, true /* is_write_commited */,
&exec_state_);
}
}

36
tools/ldb_cmd_test.cc
Unescape View File

@ -12,6 +12,8 @@ using std::string;
using std::vector;
using std::map;
namespace rocksdb {
class LdbCmdTest : public testing::Test {};
TEST_F(LdbCmdTest, HexToString) {
@ -47,6 +49,40 @@ TEST_F(LdbCmdTest, HexToStringBadInputs) {
}
}
TEST_F(LdbCmdTest, MemEnv) {
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
Options opts;
opts.env = env.get();
opts.create_if_missing = true;
DB* db = nullptr;
std::string dbname = test::TmpDir();
ASSERT_OK(DB::Open(opts, dbname, &db));
WriteOptions wopts;
for (int i = 0; i < 100; i++) {
char buf[16];
snprintf(buf, sizeof(buf), "%08d", i);
ASSERT_OK(db->Put(wopts, buf, buf));
}
FlushOptions fopts;
fopts.wait = true;
ASSERT_OK(db->Flush(fopts));
delete db;
char arg1[] = "./ldb";
char arg2[1024];
snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
char arg3[] = "dump_live_files";
char* argv[] = {arg1, arg2, arg3};
rocksdb::LDBTool tool;
tool.Run(3, argv, opts);
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();

91
tools/sst_dump_test.cc
Unescape View File

@ -38,24 +38,18 @@ static std::string MakeValue(int i) {
return key.Encode().ToString();
}
void createSST(const std::string& file_name,
const BlockBasedTableOptions& table_options) {
std::shared_ptr<rocksdb::TableFactory> tf;
tf.reset(new rocksdb::BlockBasedTableFactory(table_options));
std::unique_ptr<WritableFile> file;
Env* env = Env::Default();
EnvOptions env_options;
void createSST(const Options& opts, const std::string& file_name) {
Env* env = opts.env;
EnvOptions env_options(opts);
ReadOptions read_options;
Options opts;
const ImmutableCFOptions imoptions(opts);
const MutableCFOptions moptions(opts);
rocksdb::InternalKeyComparator ikc(opts.comparator);
std::unique_ptr<TableBuilder> tb;
std::unique_ptr<WritableFile> file;
ASSERT_OK(env->NewWritableFile(file_name, &file, env_options));
opts.table_factory = tf;
std::vector<std::unique_ptr<IntTblPropCollectorFactory> >
int_tbl_prop_collector_factories;
std::unique_ptr<WritableFileWriter> file_writer(
@ -80,8 +74,8 @@ void createSST(const std::string& file_name,
file_writer->Close();
}
void cleanup(const std::string& file_name) {
Env* env = Env::Default();
void cleanup(const Options& opts, const std::string& file_name) {
Env* env = opts.env;
env->DeleteFile(file_name);
std::string outfile_name = file_name.substr(0, file_name.length() - 4);
outfile_name.append("_dump.txt");
@ -94,8 +88,6 @@ class SSTDumpToolTest : public testing::Test {
std::string testDir_;
public:
BlockBasedTableOptions table_options_;
SSTDumpToolTest() { testDir_ = test::TmpDir(); }
~SSTDumpToolTest() {}
@ -119,88 +111,121 @@ class SSTDumpToolTest : public testing::Test {
};
TEST_F(SSTDumpToolTest, EmptyFilter) {
Options opts;
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_);
createSST(opts, file_path);
char* usage[3];
PopulateCommandArgs(file_path, "--command=raw", usage);
rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage));
ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path);
cleanup(opts, file_path);
for (int i = 0; i < 3; i++) {
delete[] usage[i];
}
}
TEST_F(SSTDumpToolTest, FilterBlock) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_);
createSST(opts, file_path);
char* usage[3];
PopulateCommandArgs(file_path, "--command=raw", usage);
rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage));
ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path);
cleanup(opts, file_path);
for (int i = 0; i < 3; i++) {
delete[] usage[i];
}
}
TEST_F(SSTDumpToolTest, FullFilterBlock) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_);
createSST(opts, file_path);
char* usage[3];
PopulateCommandArgs(file_path, "--command=raw", usage);
rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage));
ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path);
cleanup(opts, file_path);
for (int i = 0; i < 3; i++) {
delete[] usage[i];
}
}
TEST_F(SSTDumpToolTest, GetProperties) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_);
createSST(opts, file_path);
char* usage[3];
PopulateCommandArgs(file_path, "--show_properties", usage);
rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage));
ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path);
cleanup(opts, file_path);
for (int i = 0; i < 3; i++) {
delete[] usage[i];
}
}
TEST_F(SSTDumpToolTest, CompressedSizes) {
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
Options opts;
BlockBasedTableOptions table_opts;
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(file_path, table_options_);
createSST(opts, file_path);
char* usage[3];
PopulateCommandArgs(file_path, "--command=recompress", usage);
rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage));
ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(file_path);
cleanup(opts, file_path);
for (int i = 0; i < 3; i++) {
delete[] usage[i];
}
}
TEST_F(SSTDumpToolTest, MemEnv) {
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
Options opts;
opts.env = env.get();
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
createSST(opts, file_path);
char* usage[3];
PopulateCommandArgs(file_path, "--command=verify_checksum", usage);
rocksdb::SSTDumpTool tool;
ASSERT_TRUE(!tool.Run(3, usage, opts));
cleanup(opts, file_path);
for (int i = 0; i < 3; i++) {
delete[] usage[i];
}
}
} // namespace rocksdb
int main(int argc, char** argv) {

11
tools/sst_dump_tool.cc
Unescape View File

@ -43,12 +43,14 @@
namespace rocksdb {
SstFileDumper::SstFileDumper(const std::string& file_path, bool verify_checksum,
SstFileDumper::SstFileDumper(const Options& options,
const std::string& file_path, bool verify_checksum,
bool output_hex)
: file_name_(file_path),
read_num_(0),
verify_checksum_(verify_checksum),
output_hex_(output_hex),
options_(options),
ioptions_(options_),
moptions_(ColumnFamilyOptions(options_)),
internal_comparator_(BytewiseComparator()) {
@ -417,7 +419,7 @@ void print_help() {
} // namespace
int SSTDumpTool::Run(int argc, char** argv) {
int SSTDumpTool::Run(int argc, char** argv, Options options) {
const char* dir_or_file = nullptr;
uint64_t read_num = std::numeric_limits<uint64_t>::max();
std::string command;
@ -545,7 +547,7 @@ int SSTDumpTool::Run(int argc, char** argv) {
}
std::vector<std::string> filenames;
rocksdb::Env* env = rocksdb::Env::Default();
rocksdb::Env* env = options.env;
rocksdb::Status st = env->GetChildren(dir_or_file, &filenames);
bool dir = true;
if (!st.ok()) {
@ -570,7 +572,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
filename = std::string(dir_or_file) + "/" + filename;
}
rocksdb::SstFileDumper dumper(filename, verify_checksum, output_hex);
rocksdb::SstFileDumper dumper(options, filename, verify_checksum,
output_hex);
if (!dumper.getStatus().ok()) {
fprintf(stderr, "%s: %s\n", filename.c_str(),
dumper.getStatus().ToString().c_str());

4
tools/sst_dump_tool_imp.h
Unescape View File

@ -17,8 +17,8 @@ namespace rocksdb {
class SstFileDumper {
public:
explicit SstFileDumper(const std::string& file_name, bool verify_checksum,
bool output_hex);
explicit SstFileDumper(const Options& options, const std::string& file_name,
bool verify_checksum, bool output_hex);
Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from,
const std::string& from_key, bool has_to,

Write Preview
Loading…
Cancel
Save