Add compatibility test of SST ingestion (#4310)

Summary:
Test plan
```
$cd rocksdb/
$./tools/check_format_compatible.sh
```
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4310

Differential Revision: D9498125

Pulled By: riversand963

fbshipit-source-id: 83cf6992949a52199e7812bb41bc9281ac271a24

tools/check_format_compatible.sh

@@ -17,7 +17,7 @@ mkdir $input_data_path || true
 rm -rf $script_copy_dir
 cp $scriptpath $script_copy_dir -rf
 
-# Generate four random files.
+# Generate random files.
 for i in {1..6}
 do
   input_data[$i]=$input_data_path/data$i
@@ -41,10 +41,22 @@ with open('${input_data[$i]}', 'w') as f:
 EOF
 done
 
+# Generate file(s) with sorted keys.
+sorted_input_data=$input_data_path/sorted_data
+echo == Generating file with sorted keys ${sorted_input_data}
+python - <<EOF
+with open('${sorted_input_data}', 'w') as f:
+  for i in range(0,10):
+    k = str(i)
+    v = "value" + k
+    print >> f, k + " ==> " + v
+EOF
+
 declare -a backward_compatible_checkout_objs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb")
 declare -a forward_compatible_checkout_objs=("3.10.fb" "3.11.fb" "3.12.fb" "3.13.fb" "4.0.fb" "4.1.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb")
 declare -a forward_compatible_with_options_checkout_objs=("5.11.fb" "5.12.fb" "5.13.fb" "5.14.fb")
 declare -a checkout_objs=(${backward_compatible_checkout_objs[@]} ${forward_compatible_checkout_objs[@]} ${forward_compatible_with_options_checkout_objs[@]})
+declare -a extern_sst_ingestion_compatible_checkout_objs=("5.14.fb" "5.15.fb")
 
 generate_db()
 {
@@ -68,6 +80,28 @@ compare_db()
     set -e
 }
 
+write_external_sst()
+{
+    set +e
+    $script_copy_dir/write_external_sst.sh $1 $2 $3
+    if [ $? -ne 0 ]; then
+        echo ==== Error writing external SST file using data from $1 to $3 ====
+        exit 1
+    fi
+    set -e
+}
+
+ingest_external_sst()
+{
+    set +e
+    $script_copy_dir/ingest_external_sst.sh $1 $2
+    if [ $? -ne 0 ]; then
+        echo ==== Error ingesting external SST in $2 to DB at $1 ====
+        exit 1
+    fi
+    set -e
+}
+
 # Sandcastle sets us up with a remote that is just another directory on the same
 # machine and doesn't have our branches. Need to fetch them so checkout works.
 # Remote add may fail if added previously (we don't cleanup).
@@ -75,6 +109,41 @@ git remote add github_origin "https://github.com/facebook/rocksdb.git"
 set -e
 https_proxy="fwdproxy:8080" git fetch github_origin
 
+# Compatibility test for external SST file ingestion
+for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}"
+do
+  echo == Generating DB with extern SST file in "$checkout_obj" ...
+  https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj
+  make clean
+  make ldb -j32
+  write_external_sst $input_data_path $test_dir/$checkout_obj $test_dir/$checkout_obj
+  ingest_external_sst $test_dir/$checkout_obj $test_dir/$checkout_obj
+done
+
+checkout_flag=${1:-"master"}
+
+echo == Building $checkout_flag debug
+https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag
+make clean
+make ldb -j32
+compare_base_db_dir=$test_dir"/base_db_dir"
+write_external_sst $input_data_path $compare_base_db_dir $compare_base_db_dir
+ingest_external_sst $compare_base_db_dir $compare_base_db_dir
+
+for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}"
+do
+  echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag
+  git checkout $checkout_obj
+  make clean
+  make ldb -j32
+  compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 1
+  git checkout tmp-$checkout_flag
+  # Clean up
+  git branch -D $checkout_obj
+done
+
+echo == Finish compatibility test for SST ingestion.
+
 for checkout_obj in "${checkout_objs[@]}"
 do
    echo == Generating DB from "$checkout_obj" ...
@@ -87,7 +156,7 @@ done
 checkout_flag=${1:-"master"}
 
 echo == Building $checkout_flag debug
-https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag
+git checkout tmp-$checkout_flag
 make clean
 make ldb -j32
 compare_base_db_dir=$test_dir"/base_db_dir"

tools/ingest_external_sst.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+#
+#
+
+if [ "$#" -lt 2 ]; then
+  echo "usage: $BASH_SOURCE <DB Path> <External SST Dir>"
+  exit 1
+fi
+
+db_dir=$1
+external_sst_dir=$2
+
+for f in `find $external_sst_dir -name extern_sst*`
+do
+  echo == Ingesting external SST file $f to DB at $db_dir
+  ./ldb --db=$db_dir --create_if_missing ingest_extern_sst $f
+done

tools/write_external_sst.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+#
+#
+
+if [ "$#" -lt 3 ]; then
+  echo "usagee: $BASH_SOURCE <input_data_path> <DB Path> <extern SST dir>"
+  exit 1
+fi
+
+input_data_dir=$1
+db_dir=$2
+extern_sst_dir=$3
+rm -rf $db_dir
+
+set -e
+
+n=0
+
+for f in `find $input_data_dir -name sorted_data*`
+do
+  echo == Writing external SST file $f to $extern_sst_dir/extern_sst${n}
+  ./ldb --db=$db_dir --create_if_missing write_extern_sst $extern_sst_dir/extern_sst${n} < $f
+  let "n = n + 1"
+done