NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve ldb CLI option support

Browse Source 
Summary:
- Made CLI arguments take precedence over options file when both are provided. Note some of the CLI args are not settable via options file, like `--compression_max_dict_bytes`, so it's necessary to allow both ways of providing options simultaneously.
- Changed `PrepareOptionsForOpenDB` to update the proper `ColumnFamilyOptions` if one exists for the user's `--column_family_name` argument. I supported this only in the base class, `LDBCommand`, so it works for the general arguments. Will defer adding support for subcommand-specific arguments.
- Made the command fail if `--try_load_options` is provided and loading options file returns NotFound. I found the previous behavior of silently continuing confusing.
Closes https://github.com/facebook/rocksdb/pull/3144

Differential Revision: D6270544

Pulled By: ajkr

fbshipit-source-id: 7c2eac9f9b38720523d74466fb9e78db53561367
main
Andrew Kryczka 7 years ago committed by Facebook Github Bot
parent c85f8ccca3
commit ed3af9ef99
1 changed files with 46 additions and 41 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 87
      tools/ldb_cmd.cc

87
tools/ldb_cmd.cc
Unescape View File

@ -312,14 +312,10 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
}
void LDBCommand::OpenDB() {
Options opt;
bool opt_set = false;
if (!create_if_missing_ && try_load_options_) {
Status s = LoadLatestOptions(db_path_, Env::Default(), &opt,
Status s = LoadLatestOptions(db_path_, Env::Default(), &options_,
&column_families_, ignore_unknown_options_);
if (s.ok()) {
opt_set = true;
} else if (!s.IsNotFound()) {
if (!s.ok()) {
// Option file exists but load option file error.
std::string msg = s.ToString();
exec_state_ = LDBCommandExecuteResult::Failed(msg);
@ -327,9 +323,7 @@ void LDBCommand::OpenDB() {
return;
}
}
if (!opt_set) {
opt = PrepareOptionsForOpenDB();
}
options_ = PrepareOptionsForOpenDB();
if (!exec_state_.IsNotStarted()) {
return;
}
@ -343,13 +337,13 @@ void LDBCommand::OpenDB() {
"ldb doesn't support TTL DB with multiple column families");
}
if (is_read_only_) {
st = DBWithTTL::Open(opt, db_path_, &db_ttl_, 0, true);
st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true);
} else {
st = DBWithTTL::Open(opt, db_path_, &db_ttl_);
st = DBWithTTL::Open(options_, db_path_, &db_ttl_);
}
db_ = db_ttl_;
} else {
if (!opt_set && column_families_.empty()) {
if (column_families_.empty()) {
// Try to figure out column family lists
std::vector<std::string> cf_list;
st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list);
@ -360,22 +354,22 @@ void LDBCommand::OpenDB() {
if (st.ok() && cf_list.size() > 1) {
// Ignore single column family DB.
for (auto cf_name : cf_list) {
column_families_.emplace_back(cf_name, opt);
column_families_.emplace_back(cf_name, options_);
}
}
}
if (is_read_only_) {
if (column_families_.empty()) {
st = DB::OpenForReadOnly(opt, db_path_, &db_);
st = DB::OpenForReadOnly(options_, db_path_, &db_);
} else {
st = DB::OpenForReadOnly(opt, db_path_, column_families_,
st = DB::OpenForReadOnly(options_, db_path_, column_families_,
&handles_opened, &db_);
}
} else {
if (column_families_.empty()) {
st = DB::Open(opt, db_path_, &db_);
st = DB::Open(options_, db_path_, &db_);
} else {
st = DB::Open(opt, db_path_, column_families_, &handles_opened, &db_);
st = DB::Open(options_, db_path_, column_families_, &handles_opened, &db_);
}
}
}
@ -405,8 +399,6 @@ void LDBCommand::OpenDB() {
CloseDB();
}
}
options_ = opt;
}
void LDBCommand::CloseDB() {
@ -499,9 +491,19 @@ bool LDBCommand::ParseStringOption(
}
Options LDBCommand::PrepareOptionsForOpenDB() {
Options opt = options_;
opt.create_if_missing = false;
ColumnFamilyOptions* cf_opts;
auto column_families_iter =
std::find_if(column_families_.begin(), column_families_.end(),
[this](const ColumnFamilyDescriptor& cf_desc) {
return cf_desc.name == column_family_name_;
});
if (column_families_iter != column_families_.end()) {
cf_opts = &column_families_iter->options;
} else {
cf_opts = static_cast<ColumnFamilyOptions*>(&options_);
}
DBOptions* db_opts = static_cast<DBOptions*>(&options_);
db_opts->create_if_missing = false;
std::map<std::string, std::string>::const_iterator itr;
@ -530,33 +532,33 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
}
if (use_table_options) {
opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
cf_opts->table_factory.reset(NewBlockBasedTableFactory(table_options));
}
itr = option_map_.find(ARG_AUTO_COMPACTION);
if (itr != option_map_.end()) {
opt.disable_auto_compactions = ! StringToBool(itr->second);
cf_opts->disable_auto_compactions = !StringToBool(itr->second);
}
itr = option_map_.find(ARG_COMPRESSION_TYPE);
if (itr != option_map_.end()) {
std::string comp = itr->second;
if (comp == "no") {
opt.compression = kNoCompression;
cf_opts->compression = kNoCompression;
} else if (comp == "snappy") {
opt.compression = kSnappyCompression;
cf_opts->compression = kSnappyCompression;
} else if (comp == "zlib") {
opt.compression = kZlibCompression;
cf_opts->compression = kZlibCompression;
} else if (comp == "bzip2") {
opt.compression = kBZip2Compression;
cf_opts->compression = kBZip2Compression;
} else if (comp == "lz4") {
opt.compression = kLZ4Compression;
cf_opts->compression = kLZ4Compression;
} else if (comp == "lz4hc") {
opt.compression = kLZ4HCCompression;
cf_opts->compression = kLZ4HCCompression;
} else if (comp == "xpress") {
opt.compression = kXpressCompression;
cf_opts->compression = kXpressCompression;
} else if (comp == "zstd") {
opt.compression = kZSTD;
cf_opts->compression = kZSTD;
} else {
// Unknown compression.
exec_state_ =
@ -568,7 +570,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES,
compression_max_dict_bytes, exec_state_)) {
if (compression_max_dict_bytes >= 0) {
opt.compression_opts.max_dict_bytes = compression_max_dict_bytes;
cf_opts->compression_opts.max_dict_bytes = compression_max_dict_bytes;
} else {
exec_state_ = LDBCommandExecuteResult::Failed(
ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0.");
@ -579,7 +581,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE,
db_write_buffer_size, exec_state_)) {
if (db_write_buffer_size >= 0) {
opt.db_write_buffer_size = db_write_buffer_size;
db_opts->db_write_buffer_size = db_write_buffer_size;
} else {
exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE +
" must be >= 0.");
@ -590,7 +592,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size,
exec_state_)) {
if (write_buffer_size > 0) {
opt.write_buffer_size = write_buffer_size;
cf_opts->write_buffer_size = write_buffer_size;
} else {
exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE +
" must be > 0.");
@ -600,30 +602,33 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
int file_size;
if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) {
if (file_size > 0) {
opt.target_file_size_base = file_size;
cf_opts->target_file_size_base = file_size;
} else {
exec_state_ =
LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0.");
}
}
if (opt.db_paths.size() == 0) {
opt.db_paths.emplace_back(db_path_, std::numeric_limits<uint64_t>::max());
if (db_opts->db_paths.size() == 0) {
db_opts->db_paths.emplace_back(db_path_,
std::numeric_limits<uint64_t>::max());
}
int fix_prefix_len;
if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len,
exec_state_)) {
if (fix_prefix_len > 0) {
opt.prefix_extractor.reset(
cf_opts->prefix_extractor.reset(
NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len)));
} else {
exec_state_ =
LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0.");
}
}
return opt;
// TODO(ajkr): this return value doesn't reflect the CF options changed, so
// subcommands that rely on this won't see the effect of CF-related CLI args.
// Such subcommands need to be changed to properly support CFs.
return options_;
}
bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key,

Write Preview
Loading…
Cancel
Save