From 97005dbd5d2dbba7e85340645d041133167bb945 Mon Sep 17 00:00:00 2001 From: Siying Dong Date: Thu, 20 Apr 2017 10:16:13 -0700 Subject: [PATCH] tools/check_format_compatible.sh to cover option file loading too Summary: tools/check_format_compatible.sh will check a newer version of RocksDB can open option files generated by older version releases. In order to achieve that, a new parameter "--try_load_options" is added to ldb. With this parameter set, if option file exists, we load the option file and use it to open the DB. With this opiton set, we can validate option loading logic. Closes https://github.com/facebook/rocksdb/pull/2178 Differential Revision: D4914989 Pulled By: siying fbshipit-source-id: db114f7724fcb41e5e9483116d84d7c4b8389ca4 --- HISTORY.md | 1 + include/rocksdb/utilities/ldb_cmd.h | 6 +++++ tools/check_format_compatible.sh | 6 ++--- tools/ldb_cmd.cc | 42 ++++++++++++++++++++++++----- tools/ldb_cmd_impl.h | 1 - tools/ldb_test.py | 5 ++-- tools/ldb_tool.cc | 2 ++ tools/verify_random_db.sh | 12 ++++++--- 8 files changed, 60 insertions(+), 15 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index aa3c0d43b..498d9324e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,7 @@ ## Unreleased ### New Features * DB::ResetStats() to reset internal stats. +* ldb add option --try_load_options, which will open DB with its own option file. ## 5.4.0 (04/11/2017) ### Public API Change diff --git a/include/rocksdb/utilities/ldb_cmd.h b/include/rocksdb/utilities/ldb_cmd.h index aad5a5617..732e70865 100644 --- a/include/rocksdb/utilities/ldb_cmd.h +++ b/include/rocksdb/utilities/ldb_cmd.h @@ -39,6 +39,7 @@ class LDBCommand { static const std::string ARG_TTL_START; static const std::string ARG_TTL_END; static const std::string ARG_TIMESTAMP; + static const std::string ARG_TRY_LOAD_OPTIONS; static const std::string ARG_FROM; static const std::string ARG_TO; static const std::string ARG_MAX_KEYS; @@ -143,6 +144,11 @@ class LDBCommand { // If true, the kvs are output with their insert/modify timestamp in a ttl db bool timestamp_; + // If true, try to construct options from DB's option files. + bool try_load_options_; + + bool create_if_missing_; + /** * Map of options passed on the command-line. */ diff --git a/tools/check_format_compatible.sh b/tools/check_format_compatible.sh index f1576aa7e..622557688 100755 --- a/tools/check_format_compatible.sh +++ b/tools/check_format_compatible.sh @@ -58,7 +58,7 @@ generate_db() compare_db() { set +e - $script_copy_dir/verify_random_db.sh $1 $2 $3 + $script_copy_dir/verify_random_db.sh $1 $2 $3 $4 if [ $? -ne 0 ]; then echo ==== Read different content from $1 and $2 or error happened. ==== exit 1 @@ -95,7 +95,7 @@ generate_db $input_data_path $compare_base_db_dir for checkout_obj in "${checkout_objs[@]}" do echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ... - compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt + compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 done for checkout_obj in "${forward_compatible_checkout_objs[@]}" @@ -104,7 +104,7 @@ do git checkout origin/$checkout_obj make clean make ldb -j32 - compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt + compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0 done echo ==== Compatibility Test PASSED ==== diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 5b759d38c..53b66518b 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -22,6 +22,7 @@ #include "rocksdb/utilities/backupable_db.h" #include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_util.h" #include "rocksdb/write_batch.h" #include "rocksdb/write_buffer_manager.h" #include "table/scoped_arena_iterator.h" @@ -55,6 +56,7 @@ const std::string LDBCommand::ARG_TTL = "ttl"; const std::string LDBCommand::ARG_TTL_START = "start_time"; const std::string LDBCommand::ARG_TTL_END = "end_time"; const std::string LDBCommand::ARG_TIMESTAMP = "timestamp"; +const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options"; const std::string LDBCommand::ARG_FROM = "from"; const std::string LDBCommand::ARG_TO = "to"; const std::string LDBCommand::ARG_MAX_KEYS = "max_keys"; @@ -248,6 +250,12 @@ void LDBCommand::Run() { if (db_ == nullptr && !NoDBOpen()) { OpenDB(); + if (exec_state_.IsFailed() && try_load_options_) { + // We don't always return if there is a failure because a WAL file or + // manifest file can be given to "dump" command so we should continue. + // --try_load_options is not valid in those cases. + return; + } } // We'll intentionally proceed even if the DB can't be opened because users @@ -272,6 +280,8 @@ LDBCommand::LDBCommand(const std::map& options, is_value_hex_(false), is_db_ttl_(false), timestamp_(false), + try_load_options_(false), + create_if_missing_(false), option_map_(options), flags_(flags), valid_cmd_line_options_(valid_cmd_line_options) { @@ -291,10 +301,28 @@ LDBCommand::LDBCommand(const std::map& options, is_value_hex_ = IsValueHex(options, flags); is_db_ttl_ = IsFlagPresent(flags, ARG_TTL); timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP); + try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS); } void LDBCommand::OpenDB() { - Options opt = PrepareOptionsForOpenDB(); + Options opt; + bool opt_set = false; + if (!create_if_missing_ && try_load_options_) { + Status s = + LoadLatestOptions(db_path_, Env::Default(), &opt, &column_families_); + if (s.ok()) { + opt_set = true; + } else if (!s.IsNotFound()) { + // Option file exists but load option file error. + std::string msg = s.ToString(); + exec_state_ = LDBCommandExecuteResult::Failed(msg); + db_ = nullptr; + return; + } + } + if (!opt_set) { + opt = PrepareOptionsForOpenDB(); + } if (!exec_state_.IsNotStarted()) { return; } @@ -314,7 +342,7 @@ void LDBCommand::OpenDB() { } db_ = db_ttl_; } else { - if (column_families_.empty()) { + if (!opt_set && column_families_.empty()) { // Try to figure out column family lists std::vector cf_list; st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list); @@ -408,6 +436,7 @@ std::vector LDBCommand::BuildCmdLineOptions( ARG_WRITE_BUFFER_SIZE, ARG_FILE_SIZE, ARG_FIX_PREFIX_LEN, + ARG_TRY_LOAD_OPTIONS, ARG_CF_NAME}; ret.insert(ret.end(), options.begin(), options.end()); return ret; @@ -807,7 +836,6 @@ DBLoaderCommand::DBLoaderCommand( BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL, ARG_BULK_LOAD, ARG_COMPACT})), - create_if_missing_(false), disable_wal_(false), bulk_load_(false), compact_(false) { @@ -1656,7 +1684,7 @@ void ReduceDBLevelsCommand::DoCommand() { old_levels_ = old_level_num; OpenDB(); - if (!db_) { + if (exec_state_.IsFailed()) { return; } // Compact the whole DB to put all files to the highest level. @@ -2140,6 +2168,7 @@ BatchPutCommand::BatchPutCommand( is_value_hex_ ? HexToString(value) : value)); } } + create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); } void BatchPutCommand::Help(std::string& ret) { @@ -2172,7 +2201,7 @@ void BatchPutCommand::DoCommand() { Options BatchPutCommand::PrepareOptionsForOpenDB() { Options opt = LDBCommand::PrepareOptionsForOpenDB(); - opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); + opt.create_if_missing = create_if_missing_; return opt; } @@ -2424,6 +2453,7 @@ PutCommand::PutCommand(const std::vector& params, if (is_value_hex_) { value_ = HexToString(value_); } + create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); } void PutCommand::Help(std::string& ret) { @@ -2449,7 +2479,7 @@ void PutCommand::DoCommand() { Options PutCommand::PrepareOptionsForOpenDB() { Options opt = LDBCommand::PrepareOptionsForOpenDB(); - opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); + opt.create_if_missing = create_if_missing_; return opt; } diff --git a/tools/ldb_cmd_impl.h b/tools/ldb_cmd_impl.h index 40437f46f..e15f3aa51 100644 --- a/tools/ldb_cmd_impl.h +++ b/tools/ldb_cmd_impl.h @@ -139,7 +139,6 @@ class DBLoaderCommand : public LDBCommand { virtual Options PrepareOptionsForOpenDB() override; private: - bool create_if_missing_; bool disable_wal_; bool bulk_load_; bool compact_; diff --git a/tools/ldb_test.py b/tools/ldb_test.py index bbcdc18e1..2fa77bbb5 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -52,7 +52,6 @@ class LDBTestCase(unittest.TestCase): Allows full flexibility in testing; for example: missing db param. """ - output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" % params, shell=True) if not unexpected: @@ -508,7 +507,7 @@ class LDBTestCase(unittest.TestCase): dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) self.assertRunOK("put cf1_1 1 --create_if_missing", "OK") self.assertRunOK("put cf1_2 2 --create_if_missing", "OK") - self.assertRunOK("put cf1_3 3", "OK") + self.assertRunOK("put cf1_3 3 --try_load_options", "OK") # Given non-default column family to single CF DB. self.assertRunFAIL("get cf1_1 --column_family=two") self.assertRunOK("create_column_family two", "OK") @@ -525,6 +524,8 @@ class LDBTestCase(unittest.TestCase): self.assertRunOK("get cf1_1 --column_family=default", "1") self.assertRunOK("dump --column_family=two", "cf2_1 ==> 1\nKeys in range: 1") + self.assertRunOK("dump --column_family=two --try_load_options", + "cf2_1 ==> 1\nKeys in range: 1") self.assertRunOK("dump", "cf1_1 ==> 1\ncf1_3 ==> 3\nKeys in range: 2") self.assertRunOK("get cf2_1 --column_family=two", diff --git a/tools/ldb_tool.cc b/tools/ldb_tool.cc index 5ecdd90a4..a796115ce 100644 --- a/tools/ldb_tool.cc +++ b/tools/ldb_tool.cc @@ -42,6 +42,8 @@ void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options, ret.append(" --" + LDBCommand::ARG_TTL + " with 'put','get','scan','dump','query','batchput'" " : DB supports ttl and value is internally timestamp-suffixed\n"); + ret.append(" --" + LDBCommand::ARG_TRY_LOAD_OPTIONS + + " : Try to load option file from DB.\n"); ret.append(" --" + LDBCommand::ARG_BLOOM_BITS + "=\n"); ret.append(" --" + LDBCommand::ARG_FIX_PREFIX_LEN + "=\n"); ret.append(" --" + LDBCommand::ARG_COMPRESSION_TYPE + diff --git a/tools/verify_random_db.sh b/tools/verify_random_db.sh index 77607b673..8ff6a3fd1 100755 --- a/tools/verify_random_db.sh +++ b/tools/verify_random_db.sh @@ -7,21 +7,27 @@ scriptpath=`dirname $BASH_SOURCE` if [ "$#" -lt 2 ]; then - echo "usage: $BASH_SOURCE [dump_file_name]" + echo "usage: $BASH_SOURCE [dump_file_name] [if_try_load_options]" exit 1 fi db_dir=$1 base_db_dir=$2 dump_file_name=${3:-"dump_file.txt"} +try_load_options=${4:-"1"} db_dump=$db_dir"/"$dump_file_name base_db_dump=$base_db_dir"/"$dump_file_name +extra_param= + +if [ "$try_load_options" = "1" ]; then + extra_param=" --try_load_options " +fi set -e echo == Dumping data from $db_dir to $db_dump -./ldb dump --db=$db_dir > $db_dump +./ldb dump --db=$db_dir $extra_param > $db_dump echo == Dumping data from $base_db_dir to $base_db_dump -./ldb dump --db=$base_db_dir > $base_db_dump +./ldb dump --db=$base_db_dir $extra_param > $base_db_dump diff $db_dump $base_db_dir