Script to check whether RocksDB can read DB generated by previous releases and vice versa

Summary: Add a script, which checks out changes from a list of tags, build them and load the same data into it. In the last, checkout the target build and make sure it can successfully open DB and read all the data. It is implemented through ldb tool, because ldb tool is available from all previous builds so that we don't have to cross build anything.

Test Plan: Run the script.

Reviewers: yhchiang, rven, anthony, kradhakrishnan, igor

Reviewed By: igor

Subscribers: leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D36639

tools/check_format_compatible.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+#
+# A shell script to load some pre generated data file to a DB using ldb tool
+# ./ldb needs to be avaible to be executed.
+#
+# Usage: <SCRIPT> [checkout]
+# `checkout` can be a tag, commit or branch name. Will build using it and check DBs generated by all previous tags can be opened by it.
+# Return value 0 means all regression tests pass. 1 if not pass.
+
+scriptpath=`dirname $BASH_SOURCE`
+test_dir=${TEST_TMPDIR:-"/tmp"}"/format_compatible_check"
+script_copy_dir=$test_dir"/script_copy"
+input_data_path=$test_dir"/test_data_input/"
+
+mkdir $test_dir || true
+mkdir $input_data_path || true
+rm -rf $script_copy_dir
+cp $scriptpath $script_copy_dir -rf
+
+# Generate four random files.
+for i in {1..6}
+do
+  input_data[$i]=$input_data_path/data$i
+  echo == Generating random input file ${input_data[$i]}
+  python - <<EOF
+import random
+random.seed($i)
+symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
+with open('${input_data[$i]}', 'w') as f:
+  for i in range(1,1024):
+    k = ""
+    for j in range(1, random.randint(1,32)):
+      k=k + symbols[random.randint(0, len(symbols) - 1)]
+    vb = ""
+    for j in range(1, random.randint(0,128)):
+      vb = vb + symbols[random.randint(0, len(symbols) - 1)]
+    v = ""
+    for j in range(1, random.randint(1, 5)):
+      v = v + vb
+    print >> f, k + " ==> " + v
+EOF
+done
+
+# v2.1 or older doesn't pass the debug build but OK with release build
+declare -a need_release_tags=("v1.5.7" "v2.1")
+declare -a tags=("v2.5" "v2.4" "v2.3" "v2.2" "v2.8" "v3.0" "v3.1" "v3.2" "v3.3" "v3.4" "rocksdb-3.5.1" "rocksdb-3.6.2" "rocksdb-3.7" "rocksdb-3.8" "rocksdb-3.9" "v3.10")
+declare -a forward_compatible_tags=("rocksdb-3.8" "rocksdb-3.9" "v3.10")
+
+generate_db()
+{
+    set +e
+    $script_copy_dir/generate_random_db.sh $1 $2
+    if [ $? -ne 0 ]; then
+        echo ==== Error loading data from $2 to $1 ====
+        exit 1
+    fi
+    set -e
+}
+
+compare_db()
+{
+    set +e
+    $script_copy_dir/verify_random_db.sh $1 $2 $3
+    if [ $? -ne 0 ]; then
+        echo ==== Read different content from $1 and $2 or error happened. ====
+        exit 1
+    fi
+    set -e
+}
+
+set -e
+for tag in "${tags[@]}" "${need_release_tags[@]}"
+do
+   echo == Generating DB from "$tag" ...
+   git checkout $tag
+   make clean
+   make ldb -j32
+   generate_db $input_data_path $test_dir/$tag
+done
+
+checkout_flag=${1:-"master"}
+
+echo == Building $checkout_flag debug
+git checkout $checkout_flag
+make clean
+make ldb -j32
+compare_base_db_dir=$test_dir"/base_db_dir"
+echo == Generate compare base DB to $compare_base_db_dir
+generate_db $input_data_path $compare_base_db_dir
+
+for tag in "${tags[@]}"
+do
+   echo == Opening DB from "$tag" using debug build of $checkout_flag ...
+   compare_db $test_dir/$tag $compare_base_db_dir db_dump.txt
+done
+
+echo == Building $checkout_flag release
+git checkout $checkout_flag
+make release
+for tag in "${need_release_tags[@]}"
+do
+   echo == Opening DB generated by "$tag" using release build of $checkout_flag ...
+   compare_db $test_dir/$tag $compare_base_db_dir db_dump.txt
+done
+
+for tag in "${forward_compatible_tags[@]}"
+do
+   echo == Build "$tag" and try to open DB generated using $checkout_flag...
+   git checkout $tag
+   make clean
+   make ldb -j32
+   compare_db $test_dir/$tag $compare_base_db_dir forward_${tag}_dump.txt
+done
+
+echo ==== Compatibility Test PASSED ====

tools/generate_random_db.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# A shell script to load some pre generated data file to a DB using ldb tool
+# ./ldb needs to be avaible to be executed.
+#
+# Usage: <SCRIPT> <input_data_path> <DB Path>
+
+if [ "$#" -lt 2 ]; then
+  echo "usage: $BASH_SOURCE <input_data_path> <DB Path>"
+  exit 1
+fi
+
+input_data_dir=$1
+db_dir=$2
+rm -rf $db_dir
+
+echo == Loading data from $input_data_dir to $db_dir
+
+declare -a compression_opts=("no" "snappy" "zlib" "bzip2")
+
+set -e
+
+n=0
+
+for f in `ls -1 $input_data_dir`
+do
+  echo == Loading $f with compression ${compression_opts[n % 4]}
+  ./ldb load --db=$db_dir --compression_type=${compression_opts[n % 4]} --bloom_bits=10 --auto_compaction=false --create_if_missing < $input_data_dir/$f
+  let "n = n + 1"
+done

tools/verify_random_db.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# A shell script to verify DB generated by generate_random_db.sh cannot opened and read correct data.
+# ./ldb needs to be avaible to be executed.
+#
+# Usage: <SCRIPT> <DB Path>
+
+scriptpath=`dirname $BASH_SOURCE`
+if [ "$#" -lt 2 ]; then
+  echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name]"
+  exit 1
+fi
+
+db_dir=$1
+base_db_dir=$2
+dump_file_name=${3:-"dump_file.txt"}
+db_dump=$db_dir"/"$dump_file_name
+base_db_dump=$base_db_dir"/"$dump_file_name
+
+set -e
+echo == Dumping data from $db_dir to $db_dump
+./ldb dump --db=$db_dir > $db_dump
+
+echo == Dumping data from $base_db_dir to $base_db_dump
+./ldb dump --db=$base_db_dir > $base_db_dump
+
+diff $db_dump $base_db_dir