NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Allow ignoring unknown options when loading options from a file

Browse Source 
Summary:
Added a flag, `ignore_unknown_options`, to skip unknown options when loading an options file (using `LoadLatestOptions`/`LoadOptionsFromFile`) or while verifying options (using `CheckOptionsCompatibility`). This will help in downgrading the db to an older version.

Also added `--ignore_unknown_options` flag to ldb

**Example Use case:**
In MyRocks, if copying from newer version to older version, it is often impossible to start because of new RocksDB options that don't exist in older version, even though data format is compatible.
MyRocks uses these load and verify functions in [ha_rocksdb.cc::check_rocksdb_options_compatibility](e004fd9f41/storage/rocksdb/ha_rocksdb.cc (L3348-L3401)).

**Test Plan:**
Updated the unit tests.
`make check`

ldb:
$ ./ldb --db=/tmp/test_db --create_if_missing put a1 b1
OK

Now edit /tmp/test_db/<OPTIONS-file> and add an unknown option.

Try loading the options now, and it fails:
$ ./ldb --db=/tmp/test_db --try_load_options get a1
Failed: Invalid argument: Unrecognized option DBOptions:: abcd

Passes with the new --ignore_unknown_options flag
$ ./ldb --db=/tmp/test_db --try_load_options --ignore_unknown_options get a1
b1
Closes https://github.com/facebook/rocksdb/pull/2423

Differential Revision: D5212091

Pulled By: sagar0

fbshipit-source-id: 2ec17636feb47dc0351b53a77e5f15ef7cbf2ca7
main
Sagar Vemuri 8 years ago committed by Facebook Github Bot
parent 6b5a5dc5d8
commit 89ad9f3adb
11 changed files with 190 additions and 56 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 20
      include/rocksdb/convenience.h
  2. 3
      include/rocksdb/utilities/ldb_cmd.h
  3. 13
      include/rocksdb/utilities/options_util.h
  4. 62
      options/options_helper.cc
  5. 9
      options/options_helper.h
  6. 22
      options/options_parser.cc
  7. 12
      options/options_parser.h
  8. 69
      options/options_test.cc
  9. 9
      tools/ldb_cmd.cc
  10. 2
      tools/ldb_tool.cc
  11. 17
      utilities/options/options_util.cc

20
include/rocksdb/convenience.h
Unescape View File

@ -161,12 +161,15 @@ namespace rocksdb {
// @param input_strings_escaped when set to true, each escaped characters // @param input_strings_escaped when set to true, each escaped characters
// prefixed by '\' in the values of the opts_map will be further converted // prefixed by '\' in the values of the opts_map will be further converted
// back to the raw string before assigning to the associated options. // back to the raw string before assigning to the associated options.
// @param ignore_unknown_options when set to true, unknown options are ignored
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating // @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_options" will be set to "base_options". // error will be returned, and "new_options" will be set to "base_options".
Status GetColumnFamilyOptionsFromMap( Status GetColumnFamilyOptionsFromMap(
const ColumnFamilyOptions& base_options, const ColumnFamilyOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
ColumnFamilyOptions* new_options, bool input_strings_escaped = false); ColumnFamilyOptions* new_options, bool input_strings_escaped = false,
bool ignore_unknown_options = false);
// Take a default DBOptions "base_options" in addition to a // Take a default DBOptions "base_options" in addition to a
// map "opts_map" of option name to option value to construct the new // map "opts_map" of option name to option value to construct the new
@ -189,12 +192,15 @@ Status GetColumnFamilyOptionsFromMap(
// @param input_strings_escaped when set to true, each escaped characters // @param input_strings_escaped when set to true, each escaped characters
// prefixed by '\' in the values of the opts_map will be further converted // prefixed by '\' in the values of the opts_map will be further converted
// back to the raw string before assigning to the associated options. // back to the raw string before assigning to the associated options.
// @param ignore_unknown_options when set to true, unknown options are ignored
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating // @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_options" will be set to "base_options". // error will be returned, and "new_options" will be set to "base_options".
Status GetDBOptionsFromMap( Status GetDBOptionsFromMap(
const DBOptions& base_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
DBOptions* new_options, bool input_strings_escaped = false); DBOptions* new_options, bool input_strings_escaped = false,
bool ignore_unknown_options = false);
// Take a default BlockBasedTableOptions "table_options" in addition to a // Take a default BlockBasedTableOptions "table_options" in addition to a
// map "opts_map" of option name to option value to construct the new // map "opts_map" of option name to option value to construct the new
@ -229,6 +235,8 @@ Status GetDBOptionsFromMap(
// @param input_strings_escaped when set to true, each escaped characters // @param input_strings_escaped when set to true, each escaped characters
// prefixed by '\' in the values of the opts_map will be further converted // prefixed by '\' in the values of the opts_map will be further converted
// back to the raw string before assigning to the associated options. // back to the raw string before assigning to the associated options.
// @param ignore_unknown_options when set to true, unknown options are ignored
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating // @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_table_options" will be set to // error will be returned, and "new_table_options" will be set to
// "table_options". // "table_options".
@ -236,7 +244,7 @@ Status GetBlockBasedTableOptionsFromMap(
const BlockBasedTableOptions& table_options, const BlockBasedTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
BlockBasedTableOptions* new_table_options, BlockBasedTableOptions* new_table_options,
bool input_strings_escaped = false); bool input_strings_escaped = false, bool ignore_unknown_options = false);
// Take a default PlainTableOptions "table_options" in addition to a // Take a default PlainTableOptions "table_options" in addition to a
// map "opts_map" of option name to option value to construct the new // map "opts_map" of option name to option value to construct the new
@ -250,14 +258,16 @@ Status GetBlockBasedTableOptionsFromMap(
// @param input_strings_escaped when set to true, each escaped characters // @param input_strings_escaped when set to true, each escaped characters
// prefixed by '\' in the values of the opts_map will be further converted // prefixed by '\' in the values of the opts_map will be further converted
// back to the raw string before assigning to the associated options. // back to the raw string before assigning to the associated options.
// @param ignore_unknown_options when set to true, unknown options are ignored
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating // @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_table_options" will be set to // error will be returned, and "new_table_options" will be set to
// "table_options". // "table_options".
Status GetPlainTableOptionsFromMap( Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options, const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
PlainTableOptions* new_table_options, PlainTableOptions* new_table_options, bool input_strings_escaped = false,
bool input_strings_escaped = false); bool ignore_unknown_options = false);
// Take a string representation of option names and values, apply them into the // Take a string representation of option names and values, apply them into the
// base_options, and return the new options as a result. The string has the // base_options, and return the new options as a result. The string has the

3
include/rocksdb/utilities/ldb_cmd.h
Unescape View File

@ -42,6 +42,7 @@ class LDBCommand {
static const std::string ARG_TTL_END; static const std::string ARG_TTL_END;
static const std::string ARG_TIMESTAMP; static const std::string ARG_TIMESTAMP;
static const std::string ARG_TRY_LOAD_OPTIONS; static const std::string ARG_TRY_LOAD_OPTIONS;
static const std::string ARG_IGNORE_UNKNOWN_OPTIONS;
static const std::string ARG_FROM; static const std::string ARG_FROM;
static const std::string ARG_TO; static const std::string ARG_TO;
static const std::string ARG_MAX_KEYS; static const std::string ARG_MAX_KEYS;
@ -149,6 +150,8 @@ class LDBCommand {
// If true, try to construct options from DB's option files. // If true, try to construct options from DB's option files.
bool try_load_options_; bool try_load_options_;
bool ignore_unknown_options_;
bool create_if_missing_; bool create_if_missing_;
/** /**

13
include/rocksdb/utilities/options_util.h
Unescape View File

@ -41,6 +41,10 @@ namespace rocksdb {
// casting the return value of TableFactoroy::GetOptions() to // casting the return value of TableFactoroy::GetOptions() to
// BlockBasedTableOptions and making necessary changes. // BlockBasedTableOptions and making necessary changes.
// //
// ignore_unknown_options can be set to true if you want to ignore options
// that are from a newer version of the db, esentially for forward
// compatibility.
//
// examples/options_file_example.cc demonstrates how to use this function // examples/options_file_example.cc demonstrates how to use this function
// to open a RocksDB instance. // to open a RocksDB instance.
// //
@ -53,7 +57,8 @@ namespace rocksdb {
// @see LoadOptionsFromFile // @see LoadOptionsFromFile
Status LoadLatestOptions(const std::string& dbpath, Env* env, Status LoadLatestOptions(const std::string& dbpath, Env* env,
DBOptions* db_options, DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs); std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options = false);
// Similar to LoadLatestOptions, this function constructs the DBOptions // Similar to LoadLatestOptions, this function constructs the DBOptions
// and ColumnFamilyDescriptors based on the specified RocksDB Options file. // and ColumnFamilyDescriptors based on the specified RocksDB Options file.
@ -61,7 +66,8 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env,
// @see LoadLatestOptions // @see LoadLatestOptions
Status LoadOptionsFromFile(const std::string& options_file_name, Env* env, Status LoadOptionsFromFile(const std::string& options_file_name, Env* env,
DBOptions* db_options, DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs); std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options = false);
// Returns the latest options file name under the specified db path. // Returns the latest options file name under the specified db path.
Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, Status GetLatestOptionsFileName(const std::string& dbpath, Env* env,
@ -80,7 +86,8 @@ Status GetLatestOptionsFileName(const std::string& dbpath, Env* env,
// * merge_operator // * merge_operator
Status CheckOptionsCompatibility( Status CheckOptionsCompatibility(
const std::string& dbpath, Env* env, const DBOptions& db_options, const std::string& dbpath, Env* env, const DBOptions& db_options,
const std::vector<ColumnFamilyDescriptor>& cf_descs); const std::vector<ColumnFamilyDescriptor>& cf_descs,
bool ignore_unknown_options = false);
} // namespace rocksdb } // namespace rocksdb
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE

62
options/options_helper.cc
Unescape View File

@ -1008,7 +1008,8 @@ Status ParseDBOption(const std::string& name,
std::string ParseBlockBasedTableOption(const std::string& name, std::string ParseBlockBasedTableOption(const std::string& name,
const std::string& org_value, const std::string& org_value,
BlockBasedTableOptions* new_options, BlockBasedTableOptions* new_options,
bool input_strings_escaped = false) { bool input_strings_escaped = false,
bool ignore_unknown_options = false) {
const std::string& value = const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value; input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
if (!input_strings_escaped) { if (!input_strings_escaped) {
@ -1042,8 +1043,12 @@ std::string ParseBlockBasedTableOption(const std::string& name,
} }
const auto iter = block_based_table_type_info.find(name); const auto iter = block_based_table_type_info.find(name);
if (iter == block_based_table_type_info.end()) { if (iter == block_based_table_type_info.end()) {
if (ignore_unknown_options) {
return "";
} else {
return "Unrecognized option"; return "Unrecognized option";
} }
}
const auto& opt_info = iter->second; const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated && if (opt_info.verification != OptionVerificationType::kDeprecated &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset, !ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
@ -1056,13 +1061,18 @@ std::string ParseBlockBasedTableOption(const std::string& name,
std::string ParsePlainTableOptions(const std::string& name, std::string ParsePlainTableOptions(const std::string& name,
const std::string& org_value, const std::string& org_value,
PlainTableOptions* new_options, PlainTableOptions* new_options,
bool input_strings_escaped = false) { bool input_strings_escaped = false,
bool ignore_unknown_options = false) {
const std::string& value = const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value; input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
const auto iter = plain_table_type_info.find(name); const auto iter = plain_table_type_info.find(name);
if (iter == plain_table_type_info.end()) { if (iter == plain_table_type_info.end()) {
if (ignore_unknown_options) {
return "";
} else {
return "Unrecognized option"; return "Unrecognized option";
} }
}
const auto& opt_info = iter->second; const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated && if (opt_info.verification != OptionVerificationType::kDeprecated &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset, !ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
@ -1075,12 +1085,14 @@ std::string ParsePlainTableOptions(const std::string& name,
Status GetBlockBasedTableOptionsFromMap( Status GetBlockBasedTableOptionsFromMap(
const BlockBasedTableOptions& table_options, const BlockBasedTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
BlockBasedTableOptions* new_table_options, bool input_strings_escaped) { BlockBasedTableOptions* new_table_options, bool input_strings_escaped,
bool ignore_unknown_options) {
assert(new_table_options); assert(new_table_options);
*new_table_options = table_options; *new_table_options = table_options;
for (const auto& o : opts_map) { for (const auto& o : opts_map) {
auto error_message = ParseBlockBasedTableOption( auto error_message = ParseBlockBasedTableOption(
o.first, o.second, new_table_options, input_strings_escaped); o.first, o.second, new_table_options, input_strings_escaped,
ignore_unknown_options);
if (error_message != "") { if (error_message != "") {
const auto iter = block_based_table_type_info.find(o.first); const auto iter = block_based_table_type_info.find(o.first);
if (iter == block_based_table_type_info.end() || if (iter == block_based_table_type_info.end() ||
@ -1090,7 +1102,8 @@ Status GetBlockBasedTableOptionsFromMap(
(iter->second.verification != OptionVerificationType::kByName && (iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification != iter->second.verification !=
OptionVerificationType::kByNameAllowNull && OptionVerificationType::kByNameAllowNull &&
iter->second.verification != OptionVerificationType::kDeprecated)) { iter->second.verification !=
OptionVerificationType::kDeprecated)) {
// Restore "new_options" to the default "base_options". // Restore "new_options" to the default "base_options".
*new_table_options = table_options; *new_table_options = table_options;
return Status::InvalidArgument("Can't parse BlockBasedTableOptions:", return Status::InvalidArgument("Can't parse BlockBasedTableOptions:",
@ -1117,7 +1130,8 @@ Status GetBlockBasedTableOptionsFromString(
Status GetPlainTableOptionsFromMap( Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options, const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
PlainTableOptions* new_table_options, bool input_strings_escaped) { PlainTableOptions* new_table_options, bool input_strings_escaped,
bool ignore_unknown_options) {
assert(new_table_options); assert(new_table_options);
*new_table_options = table_options; *new_table_options = table_options;
for (const auto& o : opts_map) { for (const auto& o : opts_map) {
@ -1132,7 +1146,8 @@ Status GetPlainTableOptionsFromMap(
(iter->second.verification != OptionVerificationType::kByName && (iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification != iter->second.verification !=
OptionVerificationType::kByNameAllowNull && OptionVerificationType::kByNameAllowNull &&
iter->second.verification != OptionVerificationType::kDeprecated)) { iter->second.verification !=
OptionVerificationType::kDeprecated)) {
// Restore "new_options" to the default "base_options". // Restore "new_options" to the default "base_options".
*new_table_options = table_options; *new_table_options = table_options;
return Status::InvalidArgument("Can't parse PlainTableOptions:", return Status::InvalidArgument("Can't parse PlainTableOptions:",
@ -1229,16 +1244,19 @@ Status GetMemTableRepFactoryFromString(const std::string& opts_str,
Status GetColumnFamilyOptionsFromMap( Status GetColumnFamilyOptionsFromMap(
const ColumnFamilyOptions& base_options, const ColumnFamilyOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
ColumnFamilyOptions* new_options, bool input_strings_escaped) { ColumnFamilyOptions* new_options, bool input_strings_escaped,
bool ignore_unknown_options) {
return GetColumnFamilyOptionsFromMapInternal( return GetColumnFamilyOptionsFromMapInternal(
base_options, opts_map, new_options, input_strings_escaped); base_options, opts_map, new_options, input_strings_escaped, nullptr,
ignore_unknown_options);
} }
Status GetColumnFamilyOptionsFromMapInternal( Status GetColumnFamilyOptionsFromMapInternal(
const ColumnFamilyOptions& base_options, const ColumnFamilyOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
ColumnFamilyOptions* new_options, bool input_strings_escaped, ColumnFamilyOptions* new_options, bool input_strings_escaped,
std::vector<std::string>* unsupported_options_names) { std::vector<std::string>* unsupported_options_names,
bool ignore_unknown_options) {
assert(new_options); assert(new_options);
*new_options = base_options; *new_options = base_options;
if (unsupported_options_names) { if (unsupported_options_names) {
@ -1258,6 +1276,8 @@ Status GetColumnFamilyOptionsFromMapInternal(
// Note that we still return Status::OK in such case to maintain // Note that we still return Status::OK in such case to maintain
// the backward compatibility in the old public API defined in // the backward compatibility in the old public API defined in
// rocksdb/convenience.h // rocksdb/convenience.h
} else if (s.IsInvalidArgument() && ignore_unknown_options) {
continue;
} else { } else {
// Restore "new_options" to the default "base_options". // Restore "new_options" to the default "base_options".
*new_options = base_options; *new_options = base_options;
@ -1284,16 +1304,19 @@ Status GetColumnFamilyOptionsFromString(
Status GetDBOptionsFromMap( Status GetDBOptionsFromMap(
const DBOptions& base_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
DBOptions* new_options, bool input_strings_escaped) { DBOptions* new_options, bool input_strings_escaped,
return GetDBOptionsFromMapInternal( bool ignore_unknown_options) {
base_options, opts_map, new_options, input_strings_escaped); return GetDBOptionsFromMapInternal(base_options, opts_map, new_options,
input_strings_escaped, nullptr,
ignore_unknown_options);
} }
Status GetDBOptionsFromMapInternal( Status GetDBOptionsFromMapInternal(
const DBOptions& base_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
DBOptions* new_options, bool input_strings_escaped, DBOptions* new_options, bool input_strings_escaped,
std::vector<std::string>* unsupported_options_names) { std::vector<std::string>* unsupported_options_names,
bool ignore_unknown_options) {
assert(new_options); assert(new_options);
*new_options = base_options; *new_options = base_options;
if (unsupported_options_names) { if (unsupported_options_names) {
@ -1313,6 +1336,8 @@ Status GetDBOptionsFromMapInternal(
// Note that we still return Status::OK in such case to maintain // Note that we still return Status::OK in such case to maintain
// the backward compatibility in the old public API defined in // the backward compatibility in the old public API defined in
// rocksdb/convenience.h // rocksdb/convenience.h
} else if (s.IsInvalidArgument() && ignore_unknown_options) {
continue;
} else { } else {
// Restore "new_options" to the default "base_options". // Restore "new_options" to the default "base_options".
*new_options = base_options; *new_options = base_options;
@ -1360,12 +1385,14 @@ Status GetOptionsFromString(const Options& base_options,
Status GetTableFactoryFromMap( Status GetTableFactoryFromMap(
const std::string& factory_name, const std::string& factory_name,
const std::unordered_map<std::string, std::string>& opt_map, const std::unordered_map<std::string, std::string>& opt_map,
std::shared_ptr<TableFactory>* table_factory) { std::shared_ptr<TableFactory>* table_factory, bool ignore_unknown_options) {
Status s; Status s;
if (factory_name == BlockBasedTableFactory().Name()) { if (factory_name == BlockBasedTableFactory().Name()) {
BlockBasedTableOptions bbt_opt; BlockBasedTableOptions bbt_opt;
s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map, s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map,
&bbt_opt, true); &bbt_opt,
true, /* input_strings_escaped */
ignore_unknown_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -1374,7 +1401,8 @@ Status GetTableFactoryFromMap(
} else if (factory_name == PlainTableFactory().Name()) { } else if (factory_name == PlainTableFactory().Name()) {
PlainTableOptions pt_opt; PlainTableOptions pt_opt;
s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt, s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt,
true); true, /* input_strings_escaped */
ignore_unknown_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }

9
options/options_helper.h
Unescape View File

@ -57,7 +57,8 @@ Status GetMutableDBOptionsFromStrings(
Status GetTableFactoryFromMap( Status GetTableFactoryFromMap(
const std::string& factory_name, const std::string& factory_name,
const std::unordered_map<std::string, std::string>& opt_map, const std::unordered_map<std::string, std::string>& opt_map,
std::shared_ptr<TableFactory>* table_factory); std::shared_ptr<TableFactory>* table_factory,
bool ignore_unknown_options = false);
Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf, Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
const std::string& delimiter = "; "); const std::string& delimiter = "; ");
@ -129,7 +130,8 @@ Status GetDBOptionsFromMapInternal(
const DBOptions& base_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
DBOptions* new_options, bool input_strings_escaped, DBOptions* new_options, bool input_strings_escaped,
std::vector<std::string>* unsupported_options_names = nullptr); std::vector<std::string>* unsupported_options_names = nullptr,
bool ignore_unknown_options = false);
// In addition to its public version defined in rocksdb/convenience.h, // In addition to its public version defined in rocksdb/convenience.h,
// this further takes an optional output vector "unsupported_options_names", // this further takes an optional output vector "unsupported_options_names",
@ -138,7 +140,8 @@ Status GetColumnFamilyOptionsFromMapInternal(
const ColumnFamilyOptions& base_options, const ColumnFamilyOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
ColumnFamilyOptions* new_options, bool input_strings_escaped, ColumnFamilyOptions* new_options, bool input_strings_escaped,
std::vector<std::string>* unsupported_options_names = nullptr); std::vector<std::string>* unsupported_options_names = nullptr,
bool ignore_unknown_options = false);
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = { static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
/* /*

22
options/options_parser.cc
Unescape View File

@ -233,7 +233,8 @@ bool ReadOneLine(std::istringstream* iss, SequentialFile* seq_file,
} }
} // namespace } // namespace
Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) { Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env,
bool ignore_unknown_options) {
Reset(); Reset();
std::unique_ptr<SequentialFile> seq_file; std::unique_ptr<SequentialFile> seq_file;
@ -260,7 +261,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) {
continue; continue;
} }
if (IsSection(line)) { if (IsSection(line)) {
s = EndSection(section, title, argument, opt_map); s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
opt_map.clear(); opt_map.clear();
if (!s.ok()) { if (!s.ok()) {
return s; return s;
@ -280,7 +281,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) {
} }
} }
s = EndSection(section, title, argument, opt_map); s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
opt_map.clear(); opt_map.clear();
if (!s.ok()) { if (!s.ok()) {
return s; return s;
@ -389,10 +390,12 @@ Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
Status RocksDBOptionsParser::EndSection( Status RocksDBOptionsParser::EndSection(
const OptionSection section, const std::string& section_title, const OptionSection section, const std::string& section_title,
const std::string& section_arg, const std::string& section_arg,
const std::unordered_map<std::string, std::string>& opt_map) { const std::unordered_map<std::string, std::string>& opt_map,
bool ignore_unknown_options) {
Status s; Status s;
if (section == kOptionSectionDBOptions) { if (section == kOptionSectionDBOptions) {
s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true); s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true,
ignore_unknown_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -404,7 +407,8 @@ Status RocksDBOptionsParser::EndSection(
cf_names_.emplace_back(section_arg); cf_names_.emplace_back(section_arg);
cf_opts_.emplace_back(); cf_opts_.emplace_back();
s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map, s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map,
&cf_opts_.back(), true); &cf_opts_.back(), true,
ignore_unknown_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -423,7 +427,7 @@ Status RocksDBOptionsParser::EndSection(
s = GetTableFactoryFromMap( s = GetTableFactoryFromMap(
section_title.substr( section_title.substr(
opt_section_titles[kOptionSectionTableOptions].size()), opt_section_titles[kOptionSectionTableOptions].size()),
opt_map, &(cf_opt->table_factory)); opt_map, &(cf_opt->table_factory), ignore_unknown_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -615,10 +619,10 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
const DBOptions& db_opt, const std::vector<std::string>& cf_names, const DBOptions& db_opt, const std::vector<std::string>& cf_names,
const std::vector<ColumnFamilyOptions>& cf_opts, const std::vector<ColumnFamilyOptions>& cf_opts,
const std::string& file_name, Env* env, const std::string& file_name, Env* env,
OptionsSanityCheckLevel sanity_check_level) { OptionsSanityCheckLevel sanity_check_level, bool ignore_unknown_options) {
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
std::unique_ptr<SequentialFile> seq_file; std::unique_ptr<SequentialFile> seq_file;
Status s = parser.Parse(file_name, env); Status s = parser.Parse(file_name, env, ignore_unknown_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }

12
options/options_parser.h
Unescape View File

@ -44,7 +44,8 @@ class RocksDBOptionsParser {
~RocksDBOptionsParser() {} ~RocksDBOptionsParser() {}
void Reset(); void Reset();
Status Parse(const std::string& file_name, Env* env); Status Parse(const std::string& file_name, Env* env,
bool ignore_unknown_options = false);
static std::string TrimAndRemoveComment(const std::string& line, static std::string TrimAndRemoveComment(const std::string& line,
const bool trim_only = false); const bool trim_only = false);
@ -68,7 +69,8 @@ class RocksDBOptionsParser {
const DBOptions& db_opt, const std::vector<std::string>& cf_names, const DBOptions& db_opt, const std::vector<std::string>& cf_names,
const std::vector<ColumnFamilyOptions>& cf_opts, const std::vector<ColumnFamilyOptions>& cf_opts,
const std::string& file_name, Env* env, const std::string& file_name, Env* env,
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch,
bool ignore_unknown_options = false);
static Status VerifyDBOptions( static Status VerifyDBOptions(
const DBOptions& base_opt, const DBOptions& new_opt, const DBOptions& base_opt, const DBOptions& new_opt,
@ -103,10 +105,10 @@ class RocksDBOptionsParser {
Status ParseStatement(std::string* name, std::string* value, Status ParseStatement(std::string* name, std::string* value,
const std::string& line, const int line_num); const std::string& line, const int line_num);
Status EndSection( Status EndSection(const OptionSection section, const std::string& title,
const OptionSection section, const std::string& title,
const std::string& section_arg, const std::string& section_arg,
const std::unordered_map<std::string, std::string>& opt_map); const std::unordered_map<std::string, std::string>& opt_map,
bool ignore_unknown_options);
Status ValidityCheck(); Status ValidityCheck();

69
options/options_test.cc
Unescape View File

@ -202,12 +202,23 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
cf_options_map["write_buffer_size"] = "1"; cf_options_map["write_buffer_size"] = "1";
ASSERT_OK(GetColumnFamilyOptionsFromMap( ASSERT_OK(GetColumnFamilyOptionsFromMap(
base_cf_opt, cf_options_map, &new_cf_opt)); base_cf_opt, cf_options_map, &new_cf_opt));
cf_options_map["unknown_option"] = "1";
cf_options_map["unknown_option"] = "1";
ASSERT_NOK(GetColumnFamilyOptionsFromMap( ASSERT_NOK(GetColumnFamilyOptionsFromMap(
base_cf_opt, cf_options_map, &new_cf_opt)); base_cf_opt, cf_options_map, &new_cf_opt));
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
&new_cf_opt,
false, /* input_strings_escaped */
true /* ignore_unknown_options */));
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/));
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
kSanityLevelExactMatch /* default for VerifyCFOptions */));
DBOptions base_db_opt; DBOptions base_db_opt;
DBOptions new_db_opt; DBOptions new_db_opt;
ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
@ -248,6 +259,28 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159); ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47)); ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48)); ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
db_options_map["max_open_files"] = "hello";
ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
base_db_opt, new_db_opt, nullptr, /* new_opt_map */
kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
// unknow options should fail parsing without ignore_unknown_options = true
db_options_map["unknown_db_option"] = "1";
ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt,
false, /* input_strings_escaped */
true /* ignore_unknown_options */));
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
base_db_opt, new_db_opt, nullptr, /* new_opt_map */
kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(
base_db_opt, new_db_opt, nullptr, /* new_opt_mat */
kSanityLevelExactMatch /* default for VerifyDBOptions */));
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
@ -1068,6 +1101,40 @@ TEST_F(OptionsParserTest, DuplicateCFOptions) {
ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
} }
TEST_F(OptionsParserTest, IgnoreUnknownOptions) {
DBOptions db_opt;
db_opt.max_open_files = 12345;
db_opt.max_background_flushes = 301;
db_opt.max_total_wal_size = 1024;
ColumnFamilyOptions cf_opt;
std::string options_file_content =
"# This is a testing option string.\n"
"# Currently we only support \"#\" styled comment.\n"
"\n"
"[Version]\n"
" rocksdb_version=3.14.0\n"
" options_file_version=1\n"
"[DBOptions]\n"
" max_open_files=12345\n"
" max_background_flushes=301\n"
" max_total_wal_size=1024 # keep_log_file_num=1000\n"
" unknown_db_option1=321\n"
" unknown_db_option2=false\n"
"[CFOptions \"default\"]\n"
" unknown_cf_option1=hello\n"
"[CFOptions \"something_else\"]\n"
" unknown_cf_option2=world\n"
" # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini";
env_->WriteToNewFile(kTestFileName, options_file_content);
RocksDBOptionsParser parser;
ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
ASSERT_OK(parser.Parse(kTestFileName, env_.get(),
true /* ignore_unknown_options */));
}
TEST_F(OptionsParserTest, ParseVersion) { TEST_F(OptionsParserTest, ParseVersion) {
DBOptions db_opt; DBOptions db_opt;
db_opt.max_open_files = 12345; db_opt.max_open_files = 12345;

9
tools/ldb_cmd.cc
Unescape View File

@ -60,6 +60,8 @@ const std::string LDBCommand::ARG_TTL_START = "start_time";
const std::string LDBCommand::ARG_TTL_END = "end_time"; const std::string LDBCommand::ARG_TTL_END = "end_time";
const std::string LDBCommand::ARG_TIMESTAMP = "timestamp"; const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options"; const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
const std::string LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS =
"ignore_unknown_options";
const std::string LDBCommand::ARG_FROM = "from"; const std::string LDBCommand::ARG_FROM = "from";
const std::string LDBCommand::ARG_TO = "to"; const std::string LDBCommand::ARG_TO = "to";
const std::string LDBCommand::ARG_MAX_KEYS = "max_keys"; const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
@ -284,6 +286,7 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
is_db_ttl_(false), is_db_ttl_(false),
timestamp_(false), timestamp_(false),
try_load_options_(false), try_load_options_(false),
ignore_unknown_options_(false),
create_if_missing_(false), create_if_missing_(false),
option_map_(options), option_map_(options),
flags_(flags), flags_(flags),
@ -305,14 +308,15 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
is_db_ttl_ = IsFlagPresent(flags, ARG_TTL); is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP); timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS); try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
ignore_unknown_options_ = IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS);
} }
void LDBCommand::OpenDB() { void LDBCommand::OpenDB() {
Options opt; Options opt;
bool opt_set = false; bool opt_set = false;
if (!create_if_missing_ && try_load_options_) { if (!create_if_missing_ && try_load_options_) {
Status s = Status s = LoadLatestOptions(db_path_, Env::Default(), &opt,
LoadLatestOptions(db_path_, Env::Default(), &opt, &column_families_); &column_families_, ignore_unknown_options_);
if (s.ok()) { if (s.ok()) {
opt_set = true; opt_set = true;
} else if (!s.IsNotFound()) { } else if (!s.IsNotFound()) {
@ -440,6 +444,7 @@ std::vector<std::string> LDBCommand::BuildCmdLineOptions(
ARG_FILE_SIZE, ARG_FILE_SIZE,
ARG_FIX_PREFIX_LEN, ARG_FIX_PREFIX_LEN,
ARG_TRY_LOAD_OPTIONS, ARG_TRY_LOAD_OPTIONS,
ARG_IGNORE_UNKNOWN_OPTIONS,
ARG_CF_NAME}; ARG_CF_NAME};
ret.insert(ret.end(), options.begin(), options.end()); ret.insert(ret.end(), options.begin(), options.end());
return ret; return ret;

2
tools/ldb_tool.cc
Unescape View File

@ -46,6 +46,8 @@ void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options,
" : DB supports ttl and value is internally timestamp-suffixed\n"); " : DB supports ttl and value is internally timestamp-suffixed\n");
ret.append(" --" + LDBCommand::ARG_TRY_LOAD_OPTIONS + ret.append(" --" + LDBCommand::ARG_TRY_LOAD_OPTIONS +
" : Try to load option file from DB.\n"); " : Try to load option file from DB.\n");
ret.append(" --" + LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS +
" : Ignore unknown options when loading option file.\n");
ret.append(" --" + LDBCommand::ARG_BLOOM_BITS + "=<int,e.g.:14>\n"); ret.append(" --" + LDBCommand::ARG_BLOOM_BITS + "=<int,e.g.:14>\n");
ret.append(" --" + LDBCommand::ARG_FIX_PREFIX_LEN + "=<int,e.g.:14>\n"); ret.append(" --" + LDBCommand::ARG_FIX_PREFIX_LEN + "=<int,e.g.:14>\n");
ret.append(" --" + LDBCommand::ARG_COMPRESSION_TYPE + ret.append(" --" + LDBCommand::ARG_COMPRESSION_TYPE +

17
utilities/options/options_util.cc
Unescape View File

@ -14,9 +14,10 @@
namespace rocksdb { namespace rocksdb {
Status LoadOptionsFromFile(const std::string& file_name, Env* env, Status LoadOptionsFromFile(const std::string& file_name, Env* env,
DBOptions* db_options, DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs) { std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options) {
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
Status s = parser.Parse(file_name, env); Status s = parser.Parse(file_name, env, ignore_unknown_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -61,20 +62,22 @@ Status GetLatestOptionsFileName(const std::string& dbpath,
Status LoadLatestOptions(const std::string& dbpath, Env* env, Status LoadLatestOptions(const std::string& dbpath, Env* env,
DBOptions* db_options, DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs) { std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options) {
std::string options_file_name; std::string options_file_name;
Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, db_options,
db_options, cf_descs); cf_descs, ignore_unknown_options);
} }
Status CheckOptionsCompatibility( Status CheckOptionsCompatibility(
const std::string& dbpath, Env* env, const DBOptions& db_options, const std::string& dbpath, Env* env, const DBOptions& db_options,
const std::vector<ColumnFamilyDescriptor>& cf_descs) { const std::vector<ColumnFamilyDescriptor>& cf_descs,
bool ignore_unknown_options) {
std::string options_file_name; std::string options_file_name;
Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name);
if (!s.ok()) { if (!s.ok()) {
@ -92,7 +95,7 @@ Status CheckOptionsCompatibility(
return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
db_options, cf_names, cf_opts, dbpath + "/" + options_file_name, env, db_options, cf_names, cf_opts, dbpath + "/" + options_file_name, env,
kDefaultLevel); kDefaultLevel, ignore_unknown_options);
} }
} // namespace rocksdb } // namespace rocksdb

Write Preview
Loading…
Cancel
Save