Summary:
This diff includes an initial script running a set of benchmarks for
regression test. The script does the following things:
checkout the specified rocksdb commit (or origin/master as default)
make clean && DEBUG_LEVEL=0 make db_bench
setup test directories
run set of benchmarks and store results
Currently, the script will run couple benchmarks, store all the benchmark
output, extract micros per op and percentile information for each benchmark
and store them in a single SUMMARY.csv file. The SUMMARY.csv will make the
follow-up regression detection easier.
In addition, the current script only takes env arguments to set important
attributes of db_bench. Will follow-up with a patch that allows db_bench
to construct options from an options file.
Test Plan:
NUM_KEYS=100 ./tools/regression_test.sh
Sample SUMMARY.csv file:
commit id, benchmark, ms-per-op, p50, p75, p99, p99.9, p99.99
7e23ddf575890510e7d2fc7a79b31a1bbf317917, fillseq, 15.28, 54.66, 77.14, 5000.00, 17900.00, 18483.00
7e23ddf575890510e7d2fc7a79b31a1bbf317917, overwrite, 13.54, 57.69, 86.39, 3000.00, 15600.00, 17013.00
7e23ddf575890510e7d2fc7a79b31a1bbf317917, readrandom, 1.04, 0.80, 1.67, 293.33, 395.00, 504.00
7e23ddf575890510e7d2fc7a79b31a1bbf317917, readwhilewriting, 2.75, 1.01, 1.87, 200.00, 460.00, 485.00
7e23ddf575890510e7d2fc7a79b31a1bbf317917, deleterandom, 3.64, 48.12, 70.09, 200.00, 336.67, 347.00
7e23ddf575890510e7d2fc7a79b31a1bbf317917, seekrandom, 24.31, 391.87, 513.69, 872.73, 990.00, 1048.00
7e23ddf575890510e7d2fc7a79b31a1bbf317917, seekrandomwhilewriting, 14.02, 185.14, 294.15, 700.00, 1440.00, 1527.00
Reviewers: sdong, IslamAbdelRahman, kradhakrishnan, yiwu, andrewkr, gunnarku
Reviewed By: gunnarku
Subscribers: gunnarku, MarkCallaghan, andrewkr, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D57597
main
Yueh-Hsuan Chiang
9 years ago
parent
e1951b6f28
commit
fca5aa6fcc
@ -0,0 +1,176 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# REQUIRE: db_bench binary exists in the current directory |
||||||
|
|
||||||
|
function main { |
||||||
|
commit=${1:-"origin/master"} |
||||||
|
test_root_dir=${2:-"/tmp/rocksdb/regression_test"} |
||||||
|
|
||||||
|
init_arguments |
||||||
|
|
||||||
|
checkout_rocksdb $commit |
||||||
|
build_db_bench |
||||||
|
setup_test_directory |
||||||
|
|
||||||
|
# an additional dot indicates we share same env variables |
||||||
|
run_db_bench "fillseq" 0 |
||||||
|
run_db_bench "overwrite" |
||||||
|
run_db_bench "readrandom" |
||||||
|
run_db_bench "readwhilewriting" |
||||||
|
run_db_bench "deleterandom" |
||||||
|
run_db_bench "seekrandom" |
||||||
|
run_db_bench "seekrandomwhilewriting" |
||||||
|
|
||||||
|
echo "" |
||||||
|
echo "Benchmark completed! Results are available in $RESULT_PATH" |
||||||
|
} |
||||||
|
|
||||||
|
############################################################################ |
||||||
|
function init_arguments { |
||||||
|
K=1024 |
||||||
|
M=$((1024 * K)) |
||||||
|
G=$((1024 * M)) |
||||||
|
|
||||||
|
current_time=$(date +"%F-%H:%M:%S") |
||||||
|
RESULT_PATH=${4:-"/tmp/rocksdb/regression/results/$current_time"} |
||||||
|
COMMIT_ID=`git log | head -n1 | cut -c 8-` |
||||||
|
SUMMARY_FILE="$RESULT_PATH/SUMMARY.csv" |
||||||
|
|
||||||
|
DB_PATH=${2:-"/tmp/rocksdb/regression/db/"} |
||||||
|
WAL_PATH=${3:-"/tmp/rocksdb/regression/wal/"} |
||||||
|
NUM_THREADS=${NUM_THREADS:-16} |
||||||
|
NUM_KEYS=${NUM_KEYS:-$((1 * G))} |
||||||
|
KEY_SIZE=${KEY_SIZE:-100} |
||||||
|
VALUE_SIZE=${VALUE_SIZE:-900} |
||||||
|
CACHE_SIZE=${CACHE_SIZE:-$((1 * G))} |
||||||
|
STATISTICS=${STATISTICS:-0} |
||||||
|
COMPRESSION_RATIO=${COMPRESSION_RATIO:-0.5} |
||||||
|
HISTOGRAM=${HISTOGRAM:-1} |
||||||
|
STATS_PER_INTERVAL=${STATS_PER_INTERVAL:-1} |
||||||
|
STATS_INTERVAL_SECONDS=${STATS_INTERVAL_SECONDS:-60} |
||||||
|
MAX_BACKGROUND_FLUSHES=${MAX_BACKGROUND_FLUSHES:-4} |
||||||
|
MAX_BACKGROUND_COMPACTIONS=${MAX_BACKGROUND_COMPACTIONS:-16} |
||||||
|
SEEK_NEXTS=${SEEK_NEXTS:-10} |
||||||
|
SEED=${SEED:-$( date +%s )} |
||||||
|
} |
||||||
|
|
||||||
|
# $1 --- benchmark name |
||||||
|
# $2 --- use_existing_db (optional) |
||||||
|
function run_db_bench { |
||||||
|
USE_EXISTING_DB=${2:-1} |
||||||
|
echo "" |
||||||
|
echo "=======================================================================" |
||||||
|
echo "Benchmark $1" |
||||||
|
echo "=======================================================================" |
||||||
|
echo "" |
||||||
|
db_bench_error=0 |
||||||
|
cmd="(./db_bench --benchmarks=$1 --db=$DB_PATH --wal_dir=$WAL_PATH \ |
||||||
|
--use_existing_db=$USE_EXISTING_DB \ |
||||||
|
--threads=$NUM_THREADS \ |
||||||
|
--num=$NUM_KEYS \ |
||||||
|
--key_size=$KEY_SIZE \ |
||||||
|
--value_size=$VALUE_SIZE \ |
||||||
|
--cache_size=$CACHE_SIZE \ |
||||||
|
--statistics=$STATISTICS \ |
||||||
|
--compression_ratio=$COMPRESSION_RATIO \ |
||||||
|
--histogram=$HISTOGRAM \ |
||||||
|
--seek_nexts=$SEEK_NEXTS \ |
||||||
|
--stats_per_interval=$STATS_PER_INTERVAL \ |
||||||
|
--stats_interval_seconds=$STATS_INTERVAL_SECONDS \ |
||||||
|
--max_background_flushes=$MAX_BACKGROUND_FLUSHES \ |
||||||
|
--max_background_compactions=$MAX_BACKGROUND_COMPACTIONS \ |
||||||
|
--seed=$SEED \ |
||||||
|
2>&1 || db_bench_error=1) | tee -a $RESULT_PATH/$1" |
||||||
|
echo $cmd |
||||||
|
eval $cmd |
||||||
|
exit_on_error $db_bench_error |
||||||
|
|
||||||
|
update_report "$1" "$RESULT_PATH/$1" |
||||||
|
} |
||||||
|
|
||||||
|
# $1 --- name of the benchmark |
||||||
|
# $2 --- the filename of the output log of db_bench |
||||||
|
function update_report { |
||||||
|
main_result=`cat $2 | grep $1` |
||||||
|
perc_statement=`cat $2 | grep Percentile` |
||||||
|
|
||||||
|
# Obtain micros / op |
||||||
|
main_pattern="$1"'[[:blank:]]+:[[:blank:]]+([0-9\.]+)[[:blank:]]+micros/op' |
||||||
|
[[ $main_result =~ $main_pattern ]] |
||||||
|
micros_op=${BASH_REMATCH[1]} |
||||||
|
|
||||||
|
# Obtain percentile information |
||||||
|
perc_pattern='Percentiles: P50: ([0-9\.]+) P75: ([0-9\.]+) P99: ([0-9\.]+) P99.9: ([0-9\.]+) P99.99: ([0-9\.]+)' |
||||||
|
[[ $perc_statement =~ $perc_pattern ]] |
||||||
|
|
||||||
|
perc[0]=${BASH_REMATCH[1]} # p50 |
||||||
|
perc[1]=${BASH_REMATCH[2]} # p75 |
||||||
|
perc[2]=${BASH_REMATCH[3]} # p99 |
||||||
|
perc[3]=${BASH_REMATCH[4]} # p99.9 |
||||||
|
perc[4]=${BASH_REMATCH[5]} # p99.99 |
||||||
|
|
||||||
|
printf "$COMMIT_ID, %30s, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f\n" \ |
||||||
|
$1 $micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \ |
||||||
|
>> $SUMMARY_FILE |
||||||
|
exit_on_error $? |
||||||
|
} |
||||||
|
|
||||||
|
function exit_on_error { |
||||||
|
if [ $1 -ne 0 ]; then |
||||||
|
echo "" |
||||||
|
echo "ERROR: Benchmark did not complete successfully. " \ |
||||||
|
"Partial results are output to $RESULT_PATH" |
||||||
|
echo "ERROR" >> $SUMMARY_FILE |
||||||
|
exit $1 |
||||||
|
fi |
||||||
|
} |
||||||
|
|
||||||
|
function checkout_rocksdb { |
||||||
|
echo "Checking out commit $1 ..." |
||||||
|
|
||||||
|
git fetch --all |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
git checkout $1 |
||||||
|
exit_on_error $? |
||||||
|
} |
||||||
|
|
||||||
|
function build_db_bench { |
||||||
|
echo "Building db_bench ..." |
||||||
|
|
||||||
|
make clean |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
DEBUG_LEVEL=0 make db_bench -j32 |
||||||
|
exit_on_error $? |
||||||
|
} |
||||||
|
|
||||||
|
function setup_test_directory { |
||||||
|
echo "Deleting old regression test directories and creating new ones" |
||||||
|
|
||||||
|
rm -rf "$DB_PATH" |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
rm -rf "$WAL_PATH" |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
rm -rf "$RESULT_PATH" |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
mkdir -p "$DB_PATH" |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
mkdir -p "$WAL_PATH" |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
mkdir -p "$RESULT_PATH" |
||||||
|
exit_on_error $? |
||||||
|
|
||||||
|
printf "%40s, %30s, %10s, %10s, %10s, %10s, %10s, %10s\n" \ |
||||||
|
"commit id" "benchmark" "ms-per-op" "p50" "p75" "p99" "p99.9" "p99.99" \ |
||||||
|
$micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \ |
||||||
|
>> $SUMMARY_FILE |
||||||
|
} |
||||||
|
|
||||||
|
############################################################################ |
||||||
|
|
||||||
|
main $@ |
||||||
Loading…
Reference in new issue