NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial script for the new regression test

Browse Source 
Summary:
This diff includes an initial script running a set of benchmarks for
regression test.  The script does the following things:

  checkout the specified rocksdb commit (or origin/master as default)
  make clean && DEBUG_LEVEL=0 make db_bench
  setup test directories
  run set of benchmarks and store results

Currently, the script will run couple benchmarks, store all the benchmark
output, extract micros per op and percentile information for each benchmark
and store them in a single SUMMARY.csv file.  The SUMMARY.csv will make the
follow-up regression detection easier.

In addition, the current script only takes env arguments to set important
attributes of db_bench.  Will follow-up with a patch that allows db_bench
to construct options from an options file.

Test Plan:
NUM_KEYS=100 ./tools/regression_test.sh

  Sample SUMMARY.csv file:

                                     commit id,                      benchmark,  ms-per-op,        p50,        p75,        p99,      p99.9,     p99.99
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                        fillseq,      15.28,      54.66,      77.14,    5000.00,   17900.00,   18483.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                      overwrite,      13.54,      57.69,      86.39,    3000.00,   15600.00,   17013.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                     readrandom,       1.04,       0.80,       1.67,     293.33,     395.00,     504.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,               readwhilewriting,       2.75,       1.01,       1.87,     200.00,     460.00,     485.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                   deleterandom,       3.64,      48.12,      70.09,     200.00,     336.67,     347.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                     seekrandom,      24.31,     391.87,     513.69,     872.73,     990.00,    1048.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,         seekrandomwhilewriting,      14.02,     185.14,     294.15,     700.00,    1440.00,    1527.00

Reviewers: sdong, IslamAbdelRahman, kradhakrishnan, yiwu, andrewkr, gunnarku

Reviewed By: gunnarku

Subscribers: gunnarku, MarkCallaghan, andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57597
main
Yueh-Hsuan Chiang 10 years ago
parent e1951b6f28
commit fca5aa6fcc
4 changed files with 179 additions and 3 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      tools/benchmark.sh
  2. 2
      tools/db_bench.cc
  3. 2
      tools/db_bench_tool.cc
  4. 176
      tools/regression_test.sh

2
tools/benchmark.sh
Unescape View File

@ -54,7 +54,7 @@ compression_type=${COMPRESSION_TYPE:-snappy}
duration=${DURATION:-0}
num_keys=${NUM_KEYS:-$((1 * G))}
key_size=20
key_size=${KEY_SIZE:-20}
value_size=${VALUE_SIZE:-400}
block_size=${BLOCK_SIZE:-8192}

2
tools/db_bench.cc
Unescape View File

@ -19,5 +19,5 @@ int main() {
}
#else
#include <rocksdb/db_bench_tool.h>
int main(int argc, char** argv) { rocksdb::db_bench_tool(argc, argv); }
int main(int argc, char** argv) { return rocksdb::db_bench_tool(argc, argv); }
#endif // GFLAGS

2
tools/db_bench_tool.cc
Unescape View File

@ -1915,8 +1915,8 @@ class Benchmark {
if (!SanityCheck()) {
exit(1);
}
PrintHeader();
Open(&open_options_);
PrintHeader();
std::stringstream benchmark_stream(FLAGS_benchmarks);
std::string name;
while (std::getline(benchmark_stream, name, ',')) {

176
tools/regression_test.sh
Unescape View File

@ -0,0 +1,176 @@
#!/bin/bash
# REQUIRE: db_bench binary exists in the current directory
function main {
commit=${1:-"origin/master"}
test_root_dir=${2:-"/tmp/rocksdb/regression_test"}
init_arguments
checkout_rocksdb $commit
build_db_bench
setup_test_directory
# an additional dot indicates we share same env variables
run_db_bench "fillseq" 0
run_db_bench "overwrite"
run_db_bench "readrandom"
run_db_bench "readwhilewriting"
run_db_bench "deleterandom"
run_db_bench "seekrandom"
run_db_bench "seekrandomwhilewriting"
echo ""
echo "Benchmark completed! Results are available in $RESULT_PATH"
}
############################################################################
function init_arguments {
K=1024
M=$((1024 * K))
G=$((1024 * M))
current_time=$(date +"%F-%H:%M:%S")
RESULT_PATH=${4:-"/tmp/rocksdb/regression/results/$current_time"}
COMMIT_ID=`git log | head -n1 | cut -c 8-`
SUMMARY_FILE="$RESULT_PATH/SUMMARY.csv"
DB_PATH=${2:-"/tmp/rocksdb/regression/db/"}
WAL_PATH=${3:-"/tmp/rocksdb/regression/wal/"}
NUM_THREADS=${NUM_THREADS:-16}
NUM_KEYS=${NUM_KEYS:-$((1 * G))}
KEY_SIZE=${KEY_SIZE:-100}
VALUE_SIZE=${VALUE_SIZE:-900}
CACHE_SIZE=${CACHE_SIZE:-$((1 * G))}
STATISTICS=${STATISTICS:-0}
COMPRESSION_RATIO=${COMPRESSION_RATIO:-0.5}
HISTOGRAM=${HISTOGRAM:-1}
STATS_PER_INTERVAL=${STATS_PER_INTERVAL:-1}
STATS_INTERVAL_SECONDS=${STATS_INTERVAL_SECONDS:-60}
MAX_BACKGROUND_FLUSHES=${MAX_BACKGROUND_FLUSHES:-4}
MAX_BACKGROUND_COMPACTIONS=${MAX_BACKGROUND_COMPACTIONS:-16}
SEEK_NEXTS=${SEEK_NEXTS:-10}
SEED=${SEED:-$( date +%s )}
}
# $1 --- benchmark name
# $2 --- use_existing_db (optional)
function run_db_bench {
USE_EXISTING_DB=${2:-1}
echo ""
echo "======================================================================="
echo "Benchmark $1"
echo "======================================================================="
echo ""
db_bench_error=0
cmd="(./db_bench --benchmarks=$1 --db=$DB_PATH --wal_dir=$WAL_PATH \
--use_existing_db=$USE_EXISTING_DB \
--threads=$NUM_THREADS \
--num=$NUM_KEYS \
--key_size=$KEY_SIZE \
--value_size=$VALUE_SIZE \
--cache_size=$CACHE_SIZE \
--statistics=$STATISTICS \
--compression_ratio=$COMPRESSION_RATIO \
--histogram=$HISTOGRAM \
--seek_nexts=$SEEK_NEXTS \
--stats_per_interval=$STATS_PER_INTERVAL \
--stats_interval_seconds=$STATS_INTERVAL_SECONDS \
--max_background_flushes=$MAX_BACKGROUND_FLUSHES \
--max_background_compactions=$MAX_BACKGROUND_COMPACTIONS \
--seed=$SEED \
2>&1 || db_bench_error=1) | tee -a $RESULT_PATH/$1"
echo $cmd
eval $cmd
exit_on_error $db_bench_error
update_report "$1" "$RESULT_PATH/$1"
}
# $1 --- name of the benchmark
# $2 --- the filename of the output log of db_bench
function update_report {
main_result=`cat $2 | grep $1`
perc_statement=`cat $2 | grep Percentile`
# Obtain micros / op
main_pattern="$1"'[[:blank:]]+:[[:blank:]]+([0-9\.]+)[[:blank:]]+micros/op'
[[ $main_result =~ $main_pattern ]]
micros_op=${BASH_REMATCH[1]}
# Obtain percentile information
perc_pattern='Percentiles: P50: ([0-9\.]+) P75: ([0-9\.]+) P99: ([0-9\.]+) P99.9: ([0-9\.]+) P99.99: ([0-9\.]+)'
[[ $perc_statement =~ $perc_pattern ]]
perc[0]=${BASH_REMATCH[1]} # p50
perc[1]=${BASH_REMATCH[2]} # p75
perc[2]=${BASH_REMATCH[3]} # p99
perc[3]=${BASH_REMATCH[4]} # p99.9
perc[4]=${BASH_REMATCH[5]} # p99.99
printf "$COMMIT_ID, %30s, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f\n" \
$1 $micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \
>> $SUMMARY_FILE
exit_on_error $?
}
function exit_on_error {
if [ $1 -ne 0 ]; then
echo ""
echo "ERROR: Benchmark did not complete successfully. " \
"Partial results are output to $RESULT_PATH"
echo "ERROR" >> $SUMMARY_FILE
exit $1
fi
}
function checkout_rocksdb {
echo "Checking out commit $1 ..."
git fetch --all
exit_on_error $?
git checkout $1
exit_on_error $?
}
function build_db_bench {
echo "Building db_bench ..."
make clean
exit_on_error $?
DEBUG_LEVEL=0 make db_bench -j32
exit_on_error $?
}
function setup_test_directory {
echo "Deleting old regression test directories and creating new ones"
rm -rf "$DB_PATH"
exit_on_error $?
rm -rf "$WAL_PATH"
exit_on_error $?
rm -rf "$RESULT_PATH"
exit_on_error $?
mkdir -p "$DB_PATH"
exit_on_error $?
mkdir -p "$WAL_PATH"
exit_on_error $?
mkdir -p "$RESULT_PATH"
exit_on_error $?
printf "%40s, %30s, %10s, %10s, %10s, %10s, %10s, %10s\n" \
"commit id" "benchmark" "ms-per-op" "p50" "p75" "p99" "p99.9" "p99.99" \
$micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \
>> $SUMMARY_FILE
}
############################################################################
main $@
Write Preview
Loading…
Cancel
Save