tools/check_format_compatible.sh to cover option file loading too

Summary:
tools/check_format_compatible.sh will check a newer version of RocksDB can open option files generated by older version releases. In order to achieve that, a new parameter "--try_load_options" is added to ldb. With this parameter set, if option file exists, we load the option file and use it to open the DB. With this opiton set, we can validate option loading logic.
Closes https://github.com/facebook/rocksdb/pull/2178

Differential Revision: D4914989

Pulled By: siying

fbshipit-source-id: db114f7724fcb41e5e9483116d84d7c4b8389ca4

HISTORY.md

@@ -2,6 +2,7 @@
 ## Unreleased
 ### New Features
 * DB::ResetStats() to reset internal stats.
+* ldb add option --try_load_options, which will open DB with its own option file.
 
 ## 5.4.0 (04/11/2017)
 ### Public API Change

include/rocksdb/utilities/ldb_cmd.h

@@ -39,6 +39,7 @@ class LDBCommand {
   static const std::string ARG_TTL_START;
   static const std::string ARG_TTL_END;
   static const std::string ARG_TIMESTAMP;
+  static const std::string ARG_TRY_LOAD_OPTIONS;
   static const std::string ARG_FROM;
   static const std::string ARG_TO;
   static const std::string ARG_MAX_KEYS;
@@ -143,6 +144,11 @@ class LDBCommand {
   // If true, the kvs are output with their insert/modify timestamp in a ttl db
   bool timestamp_;
 
+  // If true, try to construct options from DB's option files.
+  bool try_load_options_;
+
+  bool create_if_missing_;
+
   /**
    * Map of options passed on the command-line.
    */

tools/check_format_compatible.sh

@@ -58,7 +58,7 @@ generate_db()
 compare_db()
 {
     set +e
-    $script_copy_dir/verify_random_db.sh $1 $2 $3
+    $script_copy_dir/verify_random_db.sh $1 $2 $3 $4
     if [ $? -ne 0 ]; then
         echo ==== Read different content from $1 and $2 or error happened. ====
         exit 1
@@ -95,7 +95,7 @@ generate_db $input_data_path $compare_base_db_dir
 for checkout_obj in "${checkout_objs[@]}"
 do
    echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ...
-   compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt
+   compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1
 done
 
 for checkout_obj in "${forward_compatible_checkout_objs[@]}"
@@ -104,7 +104,7 @@ do
    git checkout origin/$checkout_obj
    make clean
    make ldb -j32
-   compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt
+   compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0
 done
 
 echo ==== Compatibility Test PASSED ====

tools/ldb_cmd.cc

@@ -22,6 +22,7 @@
 #include "rocksdb/utilities/backupable_db.h"
 #include "rocksdb/utilities/checkpoint.h"
 #include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_util.h"
 #include "rocksdb/write_batch.h"
 #include "rocksdb/write_buffer_manager.h"
 #include "table/scoped_arena_iterator.h"
@@ -55,6 +56,7 @@ const std::string LDBCommand::ARG_TTL = "ttl";
 const std::string LDBCommand::ARG_TTL_START = "start_time";
 const std::string LDBCommand::ARG_TTL_END = "end_time";
 const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
+const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
 const std::string LDBCommand::ARG_FROM = "from";
 const std::string LDBCommand::ARG_TO = "to";
 const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
@@ -248,6 +250,12 @@ void LDBCommand::Run() {
 
   if (db_ == nullptr && !NoDBOpen()) {
     OpenDB();
+    if (exec_state_.IsFailed() && try_load_options_) {
+      // We don't always return if there is a failure because a WAL file or
+      // manifest file can be given to "dump" command so we should continue.
+      // --try_load_options is not valid in those cases.
+      return;
+    }
   }
 
   // We'll intentionally proceed even if the DB can't be opened because users
@@ -272,6 +280,8 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
       is_value_hex_(false),
       is_db_ttl_(false),
       timestamp_(false),
+      try_load_options_(false),
+      create_if_missing_(false),
       option_map_(options),
       flags_(flags),
       valid_cmd_line_options_(valid_cmd_line_options) {
@@ -291,10 +301,28 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
   is_value_hex_ = IsValueHex(options, flags);
   is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
   timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
+  try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
 }
 
 void LDBCommand::OpenDB() {
-  Options opt = PrepareOptionsForOpenDB();
+  Options opt;
+  bool opt_set = false;
+  if (!create_if_missing_ && try_load_options_) {
+    Status s =
+        LoadLatestOptions(db_path_, Env::Default(), &opt, &column_families_);
+    if (s.ok()) {
+      opt_set = true;
+    } else if (!s.IsNotFound()) {
+      // Option file exists but load option file error.
+      std::string msg = s.ToString();
+      exec_state_ = LDBCommandExecuteResult::Failed(msg);
+      db_ = nullptr;
+      return;
+    }
+  }
+  if (!opt_set) {
+    opt = PrepareOptionsForOpenDB();
+  }
   if (!exec_state_.IsNotStarted()) {
     return;
   }
@@ -314,7 +342,7 @@ void LDBCommand::OpenDB() {
     }
     db_ = db_ttl_;
   } else {
-    if (column_families_.empty()) {
+    if (!opt_set && column_families_.empty()) {
       // Try to figure out column family lists
       std::vector<std::string> cf_list;
       st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list);
@@ -408,6 +436,7 @@ std::vector<std::string> LDBCommand::BuildCmdLineOptions(
                                   ARG_WRITE_BUFFER_SIZE,
                                   ARG_FILE_SIZE,
                                   ARG_FIX_PREFIX_LEN,
+                                  ARG_TRY_LOAD_OPTIONS,
                                   ARG_CF_NAME};
   ret.insert(ret.end(), options.begin(), options.end());
   return ret;
@@ -807,7 +836,6 @@ DBLoaderCommand::DBLoaderCommand(
           BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
                                ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL,
                                ARG_BULK_LOAD, ARG_COMPACT})),
-      create_if_missing_(false),
       disable_wal_(false),
       bulk_load_(false),
       compact_(false) {
@@ -1656,7 +1684,7 @@ void ReduceDBLevelsCommand::DoCommand() {
   old_levels_ = old_level_num;
 
   OpenDB();
-  if (!db_) {
+  if (exec_state_.IsFailed()) {
     return;
   }
   // Compact the whole DB to put all files to the highest level.
@@ -2140,6 +2168,7 @@ BatchPutCommand::BatchPutCommand(
           is_value_hex_ ? HexToString(value) : value));
     }
   }
+  create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
 }
 
 void BatchPutCommand::Help(std::string& ret) {
@@ -2172,7 +2201,7 @@ void BatchPutCommand::DoCommand() {
 
 Options BatchPutCommand::PrepareOptionsForOpenDB() {
   Options opt = LDBCommand::PrepareOptionsForOpenDB();
-  opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
+  opt.create_if_missing = create_if_missing_;
   return opt;
 }
 
@@ -2424,6 +2453,7 @@ PutCommand::PutCommand(const std::vector<std::string>& params,
   if (is_value_hex_) {
     value_ = HexToString(value_);
   }
+  create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
 }
 
 void PutCommand::Help(std::string& ret) {
@@ -2449,7 +2479,7 @@ void PutCommand::DoCommand() {
 
 Options PutCommand::PrepareOptionsForOpenDB() {
   Options opt = LDBCommand::PrepareOptionsForOpenDB();
-  opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
+  opt.create_if_missing = create_if_missing_;
   return opt;
 }
 

tools/ldb_cmd_impl.h

@@ -139,7 +139,6 @@ class DBLoaderCommand : public LDBCommand {
   virtual Options PrepareOptionsForOpenDB() override;
 
  private:
-  bool create_if_missing_;
   bool disable_wal_;
   bool bulk_load_;
   bool compact_;

tools/ldb_test.py

@@ -52,7 +52,6 @@ class LDBTestCase(unittest.TestCase):
         Allows full flexibility in testing; for example: missing db param.
 
         """
-
         output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
                             params, shell=True)
         if not unexpected:
@@ -508,7 +507,7 @@ class LDBTestCase(unittest.TestCase):
         dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
         self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
         self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
-        self.assertRunOK("put cf1_3 3", "OK")
+        self.assertRunOK("put cf1_3 3 --try_load_options", "OK")
         # Given non-default column family to single CF DB.
         self.assertRunFAIL("get cf1_1 --column_family=two")
         self.assertRunOK("create_column_family two", "OK")
@@ -525,6 +524,8 @@ class LDBTestCase(unittest.TestCase):
         self.assertRunOK("get cf1_1 --column_family=default", "1")
         self.assertRunOK("dump --column_family=two",
                          "cf2_1 ==> 1\nKeys in range: 1")
+        self.assertRunOK("dump --column_family=two --try_load_options",
+                         "cf2_1 ==> 1\nKeys in range: 1")
         self.assertRunOK("dump",
                          "cf1_1 ==> 1\ncf1_3 ==> 3\nKeys in range: 2")
         self.assertRunOK("get cf2_1 --column_family=two",

tools/ldb_tool.cc

@@ -42,6 +42,8 @@ void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options,
   ret.append("  --" + LDBCommand::ARG_TTL +
              " with 'put','get','scan','dump','query','batchput'"
              " : DB supports ttl and value is internally timestamp-suffixed\n");
+  ret.append("  --" + LDBCommand::ARG_TRY_LOAD_OPTIONS +
+             " : Try to load option file from DB.\n");
   ret.append("  --" + LDBCommand::ARG_BLOOM_BITS + "=<int,e.g.:14>\n");
   ret.append("  --" + LDBCommand::ARG_FIX_PREFIX_LEN + "=<int,e.g.:14>\n");
   ret.append("  --" + LDBCommand::ARG_COMPRESSION_TYPE +

tools/verify_random_db.sh

@@ -7,21 +7,27 @@
 
 scriptpath=`dirname $BASH_SOURCE`
 if [ "$#" -lt 2 ]; then
-  echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name]"
+  echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name] [if_try_load_options]"
   exit 1
 fi
 
 db_dir=$1
 base_db_dir=$2
 dump_file_name=${3:-"dump_file.txt"}
+try_load_options=${4:-"1"}
 db_dump=$db_dir"/"$dump_file_name
 base_db_dump=$base_db_dir"/"$dump_file_name
+extra_param=
+
+if [ "$try_load_options" = "1" ]; then
+ extra_param=" --try_load_options "
+fi
 
 set -e
 echo == Dumping data from $db_dir to $db_dump
-./ldb dump --db=$db_dir > $db_dump
+./ldb dump --db=$db_dir $extra_param > $db_dump
 
 echo == Dumping data from $base_db_dir to $base_db_dump
-./ldb dump --db=$base_db_dir > $base_db_dump
+./ldb dump --db=$base_db_dir $extra_param > $base_db_dump
 
 diff $db_dump $base_db_dir