diff --git a/HISTORY.md b/HISTORY.md index aa3c0d43b..498d9324e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,7 @@ ## Unreleased ### New Features * DB::ResetStats() to reset internal stats. +* ldb add option --try_load_options, which will open DB with its own option file. ## 5.4.0 (04/11/2017) ### Public API Change diff --git a/include/rocksdb/utilities/ldb_cmd.h b/include/rocksdb/utilities/ldb_cmd.h index aad5a5617..732e70865 100644 --- a/include/rocksdb/utilities/ldb_cmd.h +++ b/include/rocksdb/utilities/ldb_cmd.h @@ -39,6 +39,7 @@ class LDBCommand { static const std::string ARG_TTL_START; static const std::string ARG_TTL_END; static const std::string ARG_TIMESTAMP; + static const std::string ARG_TRY_LOAD_OPTIONS; static const std::string ARG_FROM; static const std::string ARG_TO; static const std::string ARG_MAX_KEYS; @@ -143,6 +144,11 @@ class LDBCommand { // If true, the kvs are output with their insert/modify timestamp in a ttl db bool timestamp_; + // If true, try to construct options from DB's option files. + bool try_load_options_; + + bool create_if_missing_; + /** * Map of options passed on the command-line. */ diff --git a/tools/check_format_compatible.sh b/tools/check_format_compatible.sh index f1576aa7e..622557688 100755 --- a/tools/check_format_compatible.sh +++ b/tools/check_format_compatible.sh @@ -58,7 +58,7 @@ generate_db() compare_db() { set +e - $script_copy_dir/verify_random_db.sh $1 $2 $3 + $script_copy_dir/verify_random_db.sh $1 $2 $3 $4 if [ $? -ne 0 ]; then echo ==== Read different content from $1 and $2 or error happened. ==== exit 1 @@ -95,7 +95,7 @@ generate_db $input_data_path $compare_base_db_dir for checkout_obj in "${checkout_objs[@]}" do echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ... - compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt + compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 done for checkout_obj in "${forward_compatible_checkout_objs[@]}" @@ -104,7 +104,7 @@ do git checkout origin/$checkout_obj make clean make ldb -j32 - compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt + compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0 done echo ==== Compatibility Test PASSED ==== diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 5b759d38c..53b66518b 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -22,6 +22,7 @@ #include "rocksdb/utilities/backupable_db.h" #include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_util.h" #include "rocksdb/write_batch.h" #include "rocksdb/write_buffer_manager.h" #include "table/scoped_arena_iterator.h" @@ -55,6 +56,7 @@ const std::string LDBCommand::ARG_TTL = "ttl"; const std::string LDBCommand::ARG_TTL_START = "start_time"; const std::string LDBCommand::ARG_TTL_END = "end_time"; const std::string LDBCommand::ARG_TIMESTAMP = "timestamp"; +const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options"; const std::string LDBCommand::ARG_FROM = "from"; const std::string LDBCommand::ARG_TO = "to"; const std::string LDBCommand::ARG_MAX_KEYS = "max_keys"; @@ -248,6 +250,12 @@ void LDBCommand::Run() { if (db_ == nullptr && !NoDBOpen()) { OpenDB(); + if (exec_state_.IsFailed() && try_load_options_) { + // We don't always return if there is a failure because a WAL file or + // manifest file can be given to "dump" command so we should continue. + // --try_load_options is not valid in those cases. + return; + } } // We'll intentionally proceed even if the DB can't be opened because users @@ -272,6 +280,8 @@ LDBCommand::LDBCommand(const std::map& options, is_value_hex_(false), is_db_ttl_(false), timestamp_(false), + try_load_options_(false), + create_if_missing_(false), option_map_(options), flags_(flags), valid_cmd_line_options_(valid_cmd_line_options) { @@ -291,10 +301,28 @@ LDBCommand::LDBCommand(const std::map& options, is_value_hex_ = IsValueHex(options, flags); is_db_ttl_ = IsFlagPresent(flags, ARG_TTL); timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP); + try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS); } void LDBCommand::OpenDB() { - Options opt = PrepareOptionsForOpenDB(); + Options opt; + bool opt_set = false; + if (!create_if_missing_ && try_load_options_) { + Status s = + LoadLatestOptions(db_path_, Env::Default(), &opt, &column_families_); + if (s.ok()) { + opt_set = true; + } else if (!s.IsNotFound()) { + // Option file exists but load option file error. + std::string msg = s.ToString(); + exec_state_ = LDBCommandExecuteResult::Failed(msg); + db_ = nullptr; + return; + } + } + if (!opt_set) { + opt = PrepareOptionsForOpenDB(); + } if (!exec_state_.IsNotStarted()) { return; } @@ -314,7 +342,7 @@ void LDBCommand::OpenDB() { } db_ = db_ttl_; } else { - if (column_families_.empty()) { + if (!opt_set && column_families_.empty()) { // Try to figure out column family lists std::vector cf_list; st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list); @@ -408,6 +436,7 @@ std::vector LDBCommand::BuildCmdLineOptions( ARG_WRITE_BUFFER_SIZE, ARG_FILE_SIZE, ARG_FIX_PREFIX_LEN, + ARG_TRY_LOAD_OPTIONS, ARG_CF_NAME}; ret.insert(ret.end(), options.begin(), options.end()); return ret; @@ -807,7 +836,6 @@ DBLoaderCommand::DBLoaderCommand( BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL, ARG_BULK_LOAD, ARG_COMPACT})), - create_if_missing_(false), disable_wal_(false), bulk_load_(false), compact_(false) { @@ -1656,7 +1684,7 @@ void ReduceDBLevelsCommand::DoCommand() { old_levels_ = old_level_num; OpenDB(); - if (!db_) { + if (exec_state_.IsFailed()) { return; } // Compact the whole DB to put all files to the highest level. @@ -2140,6 +2168,7 @@ BatchPutCommand::BatchPutCommand( is_value_hex_ ? HexToString(value) : value)); } } + create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); } void BatchPutCommand::Help(std::string& ret) { @@ -2172,7 +2201,7 @@ void BatchPutCommand::DoCommand() { Options BatchPutCommand::PrepareOptionsForOpenDB() { Options opt = LDBCommand::PrepareOptionsForOpenDB(); - opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); + opt.create_if_missing = create_if_missing_; return opt; } @@ -2424,6 +2453,7 @@ PutCommand::PutCommand(const std::vector& params, if (is_value_hex_) { value_ = HexToString(value_); } + create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); } void PutCommand::Help(std::string& ret) { @@ -2449,7 +2479,7 @@ void PutCommand::DoCommand() { Options PutCommand::PrepareOptionsForOpenDB() { Options opt = LDBCommand::PrepareOptionsForOpenDB(); - opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); + opt.create_if_missing = create_if_missing_; return opt; } diff --git a/tools/ldb_cmd_impl.h b/tools/ldb_cmd_impl.h index 40437f46f..e15f3aa51 100644 --- a/tools/ldb_cmd_impl.h +++ b/tools/ldb_cmd_impl.h @@ -139,7 +139,6 @@ class DBLoaderCommand : public LDBCommand { virtual Options PrepareOptionsForOpenDB() override; private: - bool create_if_missing_; bool disable_wal_; bool bulk_load_; bool compact_; diff --git a/tools/ldb_test.py b/tools/ldb_test.py index bbcdc18e1..2fa77bbb5 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -52,7 +52,6 @@ class LDBTestCase(unittest.TestCase): Allows full flexibility in testing; for example: missing db param. """ - output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" % params, shell=True) if not unexpected: @@ -508,7 +507,7 @@ class LDBTestCase(unittest.TestCase): dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) self.assertRunOK("put cf1_1 1 --create_if_missing", "OK") self.assertRunOK("put cf1_2 2 --create_if_missing", "OK") - self.assertRunOK("put cf1_3 3", "OK") + self.assertRunOK("put cf1_3 3 --try_load_options", "OK") # Given non-default column family to single CF DB. self.assertRunFAIL("get cf1_1 --column_family=two") self.assertRunOK("create_column_family two", "OK") @@ -525,6 +524,8 @@ class LDBTestCase(unittest.TestCase): self.assertRunOK("get cf1_1 --column_family=default", "1") self.assertRunOK("dump --column_family=two", "cf2_1 ==> 1\nKeys in range: 1") + self.assertRunOK("dump --column_family=two --try_load_options", + "cf2_1 ==> 1\nKeys in range: 1") self.assertRunOK("dump", "cf1_1 ==> 1\ncf1_3 ==> 3\nKeys in range: 2") self.assertRunOK("get cf2_1 --column_family=two", diff --git a/tools/ldb_tool.cc b/tools/ldb_tool.cc index 5ecdd90a4..a796115ce 100644 --- a/tools/ldb_tool.cc +++ b/tools/ldb_tool.cc @@ -42,6 +42,8 @@ void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options, ret.append(" --" + LDBCommand::ARG_TTL + " with 'put','get','scan','dump','query','batchput'" " : DB supports ttl and value is internally timestamp-suffixed\n"); + ret.append(" --" + LDBCommand::ARG_TRY_LOAD_OPTIONS + + " : Try to load option file from DB.\n"); ret.append(" --" + LDBCommand::ARG_BLOOM_BITS + "=\n"); ret.append(" --" + LDBCommand::ARG_FIX_PREFIX_LEN + "=\n"); ret.append(" --" + LDBCommand::ARG_COMPRESSION_TYPE + diff --git a/tools/verify_random_db.sh b/tools/verify_random_db.sh index 77607b673..8ff6a3fd1 100755 --- a/tools/verify_random_db.sh +++ b/tools/verify_random_db.sh @@ -7,21 +7,27 @@ scriptpath=`dirname $BASH_SOURCE` if [ "$#" -lt 2 ]; then - echo "usage: $BASH_SOURCE [dump_file_name]" + echo "usage: $BASH_SOURCE [dump_file_name] [if_try_load_options]" exit 1 fi db_dir=$1 base_db_dir=$2 dump_file_name=${3:-"dump_file.txt"} +try_load_options=${4:-"1"} db_dump=$db_dir"/"$dump_file_name base_db_dump=$base_db_dir"/"$dump_file_name +extra_param= + +if [ "$try_load_options" = "1" ]; then + extra_param=" --try_load_options " +fi set -e echo == Dumping data from $db_dir to $db_dump -./ldb dump --db=$db_dir > $db_dump +./ldb dump --db=$db_dir $extra_param > $db_dump echo == Dumping data from $base_db_dir to $base_db_dump -./ldb dump --db=$base_db_dir > $base_db_dump +./ldb dump --db=$base_db_dir $extra_param > $base_db_dump diff $db_dump $base_db_dir