Script to check whether RocksDB can read DB generated by previous releases and vice versa

Summary: Add a script, which checks out changes from a list of tags, build them and load the same data into it. In the last, checkout the target build and make sure it can successfully open DB and read all the data. It is implemented through ldb tool, because ldb tool is available from all previous builds so that we don't have to cross build anything.

Test Plan: Run the script.

Reviewers: yhchiang, rven, anthony, kradhakrishnan, igor

Reviewed By: igor

Subscribers: leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D36639
main
sdong 10 years ago
parent 2b019a1512
commit ee9bdd38a1
  1. 115
      tools/check_format_compatible.sh
  2. 30
      tools/generate_random_db.sh
  3. 27
      tools/verify_random_db.sh

@ -0,0 +1,115 @@
#!/bin/bash
#
# A shell script to load some pre generated data file to a DB using ldb tool
# ./ldb needs to be avaible to be executed.
#
# Usage: <SCRIPT> [checkout]
# `checkout` can be a tag, commit or branch name. Will build using it and check DBs generated by all previous tags can be opened by it.
# Return value 0 means all regression tests pass. 1 if not pass.
scriptpath=`dirname $BASH_SOURCE`
test_dir=${TEST_TMPDIR:-"/tmp"}"/format_compatible_check"
script_copy_dir=$test_dir"/script_copy"
input_data_path=$test_dir"/test_data_input/"
mkdir $test_dir || true
mkdir $input_data_path || true
rm -rf $script_copy_dir
cp $scriptpath $script_copy_dir -rf
# Generate four random files.
for i in {1..6}
do
input_data[$i]=$input_data_path/data$i
echo == Generating random input file ${input_data[$i]}
python - <<EOF
import random
random.seed($i)
symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
with open('${input_data[$i]}', 'w') as f:
for i in range(1,1024):
k = ""
for j in range(1, random.randint(1,32)):
k=k + symbols[random.randint(0, len(symbols) - 1)]
vb = ""
for j in range(1, random.randint(0,128)):
vb = vb + symbols[random.randint(0, len(symbols) - 1)]
v = ""
for j in range(1, random.randint(1, 5)):
v = v + vb
print >> f, k + " ==> " + v
EOF
done
# v2.1 or older doesn't pass the debug build but OK with release build
declare -a need_release_tags=("v1.5.7" "v2.1")
declare -a tags=("v2.5" "v2.4" "v2.3" "v2.2" "v2.8" "v3.0" "v3.1" "v3.2" "v3.3" "v3.4" "rocksdb-3.5.1" "rocksdb-3.6.2" "rocksdb-3.7" "rocksdb-3.8" "rocksdb-3.9" "v3.10")
declare -a forward_compatible_tags=("rocksdb-3.8" "rocksdb-3.9" "v3.10")
generate_db()
{
set +e
$script_copy_dir/generate_random_db.sh $1 $2
if [ $? -ne 0 ]; then
echo ==== Error loading data from $2 to $1 ====
exit 1
fi
set -e
}
compare_db()
{
set +e
$script_copy_dir/verify_random_db.sh $1 $2 $3
if [ $? -ne 0 ]; then
echo ==== Read different content from $1 and $2 or error happened. ====
exit 1
fi
set -e
}
set -e
for tag in "${tags[@]}" "${need_release_tags[@]}"
do
echo == Generating DB from "$tag" ...
git checkout $tag
make clean
make ldb -j32
generate_db $input_data_path $test_dir/$tag
done
checkout_flag=${1:-"master"}
echo == Building $checkout_flag debug
git checkout $checkout_flag
make clean
make ldb -j32
compare_base_db_dir=$test_dir"/base_db_dir"
echo == Generate compare base DB to $compare_base_db_dir
generate_db $input_data_path $compare_base_db_dir
for tag in "${tags[@]}"
do
echo == Opening DB from "$tag" using debug build of $checkout_flag ...
compare_db $test_dir/$tag $compare_base_db_dir db_dump.txt
done
echo == Building $checkout_flag release
git checkout $checkout_flag
make release
for tag in "${need_release_tags[@]}"
do
echo == Opening DB generated by "$tag" using release build of $checkout_flag ...
compare_db $test_dir/$tag $compare_base_db_dir db_dump.txt
done
for tag in "${forward_compatible_tags[@]}"
do
echo == Build "$tag" and try to open DB generated using $checkout_flag...
git checkout $tag
make clean
make ldb -j32
compare_db $test_dir/$tag $compare_base_db_dir forward_${tag}_dump.txt
done
echo ==== Compatibility Test PASSED ====

@ -0,0 +1,30 @@
#!/bin/bash
#
# A shell script to load some pre generated data file to a DB using ldb tool
# ./ldb needs to be avaible to be executed.
#
# Usage: <SCRIPT> <input_data_path> <DB Path>
if [ "$#" -lt 2 ]; then
echo "usage: $BASH_SOURCE <input_data_path> <DB Path>"
exit 1
fi
input_data_dir=$1
db_dir=$2
rm -rf $db_dir
echo == Loading data from $input_data_dir to $db_dir
declare -a compression_opts=("no" "snappy" "zlib" "bzip2")
set -e
n=0
for f in `ls -1 $input_data_dir`
do
echo == Loading $f with compression ${compression_opts[n % 4]}
./ldb load --db=$db_dir --compression_type=${compression_opts[n % 4]} --bloom_bits=10 --auto_compaction=false --create_if_missing < $input_data_dir/$f
let "n = n + 1"
done

@ -0,0 +1,27 @@
#!/bin/bash
#
# A shell script to verify DB generated by generate_random_db.sh cannot opened and read correct data.
# ./ldb needs to be avaible to be executed.
#
# Usage: <SCRIPT> <DB Path>
scriptpath=`dirname $BASH_SOURCE`
if [ "$#" -lt 2 ]; then
echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name]"
exit 1
fi
db_dir=$1
base_db_dir=$2
dump_file_name=${3:-"dump_file.txt"}
db_dump=$db_dir"/"$dump_file_name
base_db_dump=$base_db_dir"/"$dump_file_name
set -e
echo == Dumping data from $db_dir to $db_dump
./ldb dump --db=$db_dir > $db_dump
echo == Dumping data from $base_db_dir to $base_db_dump
./ldb dump --db=$base_db_dir > $base_db_dump
diff $db_dump $base_db_dir
Loading…
Cancel
Save