diff --git a/build_tools/regression_build_test.sh b/build_tools/regression_build_test.sh index b0140ef48..b0c130e3c 100755 --- a/build_tools/regression_build_test.sh +++ b/build_tools/regression_build_test.sh @@ -26,15 +26,20 @@ function cleanup { } trap cleanup EXIT -git_br=$(basename $GIT_BRANCH) + +if [ -z $GIT_BRANCH ]; then + git_br=`git rev-parse --abbrev-ref HEAD` +else + git_br=$(basename $GIT_BRANCH) +fi + if [ $git_br == "master" ]; then git_br="" else git_br="."$git_br fi -make clean -OPT=-DNDEBUG make db_bench -j$(nproc) +make release # measure fillseq + fill up the DB for overwrite benchmark ./db_bench \ @@ -45,7 +50,8 @@ OPT=-DNDEBUG make db_bench -j$(nproc) --num=$NUM \ --writes=$NUM \ --cache_size=6442450944 \ - --cache_numshardbits=6 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ --open_files=55000 \ --statistics=1 \ --histogram=1 \ @@ -60,9 +66,10 @@ OPT=-DNDEBUG make db_bench -j$(nproc) --use_existing_db=1 \ --bloom_bits=10 \ --num=$NUM \ - --writes=$((NUM / 2)) \ + --writes=$((NUM / 10)) \ --cache_size=6442450944 \ - --cache_numshardbits=6 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ --open_files=55000 \ --statistics=1 \ --histogram=1 \ @@ -80,7 +87,8 @@ OPT=-DNDEBUG make db_bench -j$(nproc) --num=$NUM \ --writes=$NUM \ --cache_size=6442450944 \ - --cache_numshardbits=6 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ --open_files=55000 \ --statistics=1 \ --histogram=1 \ @@ -96,9 +104,10 @@ OPT=-DNDEBUG make db_bench -j$(nproc) --use_existing_db=1 \ --bloom_bits=10 \ --num=$NUM \ - --reads=$NUM \ + --reads=$((NUM / 5)) \ --cache_size=6442450944 \ - --cache_numshardbits=8 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ --open_files=55000 \ --disable_seek_compaction=1 \ --statistics=1 \ @@ -106,26 +115,90 @@ OPT=-DNDEBUG make db_bench -j$(nproc) --disable_data_sync=1 \ --disable_wal=1 \ --sync=0 \ - --threads=32 > ${STAT_FILE}.readrandom + --threads=16 > ${STAT_FILE}.readrandom -# measure readrandom with 300MB block cache +# measure readrandom with 100MB block cache ./db_bench \ --benchmarks=readrandom \ --db=$DATA_DIR \ --use_existing_db=1 \ --bloom_bits=10 \ --num=$NUM \ - --reads=$NUM \ - --cache_size=314572800 \ - --cache_numshardbits=8 \ + --reads=$((NUM / 5)) \ + --cache_size=104857600 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --disable_seek_compaction=1 \ + --statistics=1 \ + --histogram=1 \ + --disable_data_sync=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readrandomsmallblockcache + +# measure readrandom with 8k data in memtable +./db_bench \ + --benchmarks=overwrite,readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$NUM \ + --reads=$((NUM / 5)) \ + --writes=512 \ + --cache_size=6442450944 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ + --write_buffer_size=1000000000 \ + --open_files=55000 \ + --disable_seek_compaction=1 \ + --statistics=1 \ + --histogram=1 \ + --disable_data_sync=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readrandom_mem_sst + + +# fill up the db for readrandom benchmark with filluniquerandom (1GB total size) +./db_bench \ + --benchmarks=filluniquerandom \ + --db=$DATA_DIR \ + --use_existing_db=0 \ + --bloom_bits=10 \ + --num=$((NUM / 4)) \ + --writes=$((NUM / 4)) \ + --cache_size=6442450944 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_data_sync=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=1 > /dev/null + +# measure readrandom after load with filluniquerandom with 6GB block cache +./db_bench \ + --benchmarks=readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$((NUM / 4)) \ + --reads=$((NUM / 4)) \ + --cache_size=6442450944 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ --open_files=55000 \ --disable_seek_compaction=1 \ + --disable_auto_compactions=1 \ --statistics=1 \ --histogram=1 \ --disable_data_sync=1 \ --disable_wal=1 \ --sync=0 \ - --threads=32 > ${STAT_FILE}.readrandomsmallblockcache + --threads=16 > ${STAT_FILE}.readrandom_filluniquerandom # measure memtable performance -- none of the data gets flushed to disk ./db_bench \ @@ -135,7 +208,8 @@ OPT=-DNDEBUG make db_bench -j$(nproc) --num=$((NUM / 10)) \ --reads=$NUM \ --cache_size=6442450944 \ - --cache_numshardbits=8 \ + --cache_numshardbits=4 \ + --table_cache_numshardbits=4 \ --write_buffer_size=1000000000 \ --open_files=55000 \ --disable_seek_compaction=1 \ @@ -145,13 +219,19 @@ OPT=-DNDEBUG make db_bench -j$(nproc) --disable_wal=1 \ --sync=0 \ --value_size=10 \ - --threads=32 > ${STAT_FILE}.memtablefillreadrandom + --threads=16 > ${STAT_FILE}.memtablefillreadrandom # send data to ods function send_to_ods { key="$1" value="$2" + if [ -z $JENKINS_HOME ]; then + # running on devbox, just print out the values + echo $1 $2 + return + fi + if [ -z "$value" ];then echo >&2 "ERROR: Key $key doesn't have a value." return @@ -180,5 +260,7 @@ send_benchmark_to_ods overwrite overwrite $STAT_FILE.overwrite send_benchmark_to_ods fillseq fillseq $STAT_FILE.fillseq send_benchmark_to_ods readrandom readrandom $STAT_FILE.readrandom send_benchmark_to_ods readrandom readrandom_smallblockcache $STAT_FILE.readrandomsmallblockcache +send_benchmark_to_ods readrandom readrandom_memtable_sst $STAT_FILE.readrandom_mem_sst +send_benchmark_to_ods readrandom readrandom_fillunique_random $STAT_FILE.readrandom_filluniquerandom send_benchmark_to_ods fillrandom memtablefillrandom $STAT_FILE.memtablefillreadrandom send_benchmark_to_ods readrandom memtablereadrandom $STAT_FILE.memtablefillreadrandom