NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
4068 Commits
4 Branches
0 Tags
169 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 3bcc072d24
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '3bcc072d24'
${ noResults }
rocksdb/build_tools/rocksdb-lego-determinator

524 lines
12 KiB
Raw Blame History

#!/bin/bash
# This script is executed by Sandcastle
# to determine next steps to run
#
# Helper variables
#
REPORT_EMAIL="'report': [
{
'type':'email',
'triggers':['fail'],
'emails':['leveldb@fb.com']
}
]"
CLEANUP_ENV="
{
'name':'Cleanup environment',
'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean',
'user':'root'
}"
ONCALL="'oncall':'leveldb@fb.com'"
DEBUG="OPT=-g"
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
ASAN="COMPILE_WITH_ASAN=1"
CLANG="USE_CLANG=1"
LITE="OPT=-DROCKSDB_LITE"
TSAN="COMPILE_WITH_TSAN=1"
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
#
# RocksDB unit test in parallel
# Currently we always have noise in our parallel runs. This job is to help
# manage the noise
#
PARALLEL_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Parallel Unit Test',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'user':'root'
},
{
'name':'Print logs',
'shell':'cat t/log-*',
'user':'root'
},
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version under gcc-4.8.1',
'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'user':'root'
},
{
'name':'Print logs',
'shell':'cat t/log-*',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB unit test
#
UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $DEBUG make J=1 check',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB unit test on gcc-4.8.1
#
UNIT_TEST_COMMANDS_481="[
{
'name':'Rocksdb Unit Test on GCC 4.8.1',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $GCC_481 $DEBUG make J=1 check',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB unit test with CLANG
#
CLANG_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug',
'shell':'$CLANG $SHM $DEBUG make J=1 check',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB analyze
#
CLANG_ANALYZE_COMMANDS="[
{
'name':'Rocksdb analyze',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'RocksDB build and analyze',
'shell':'$CLANG $SHM $DEBUG make J=1 analyze',
'user':'root'
},
],
}
]"
#
# RocksDB code coverage
#
CODE_COV_COMMANDS="[
{
'name':'Rocksdb Unit Test Code Coverage',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Build, test and collect code coverage info',
'shell':'$SHM $DEBUG make J=1 coverage',
'user':'root'
},
],
}
]"
#
# RocksDB unity
#
UNITY_COMMANDS="[
{
'name':'Rocksdb Unity',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Build, test unity test',
'shell':'$SHM $DEBUG V=1 make J=1 unity',
'user':'root'
},
],
}
]"
#
# Build RocksDB lite
#
LITE_BUILD_COMMANDS="[
{
'name':'Rocksdb Lite build',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$LITE $DEBUG make J=1 static_lib',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB stress/crash test
#
STRESS_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb Stress/Crash Test',
$ONCALL,
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug stress tests',
'shell':'$SHM $DEBUG make J=1 db_stress',
'user':'root'
},
{
'name':'Build and run RocksDB debug crash tests',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 crash_test',
'user':'root'
}
],
$REPORT_EMAIL
}
]"
#
# RocksDB test under address sanitizer
#
ASAN_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under ASAN',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under ASAN',
'shell':'set -o pipefail && $SHM $ASAN $DEBUG make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'user':'root'
}
],
$REPORT_EMAIL
}
]"
#
# RocksDB crash testing under address sanitizer
#
ASAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb crash test under ASAN',
$ONCALL,
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug asan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 asan_crash_test',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB unit test under valgrind
#
VALGRIND_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under valgrind',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit tests',
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG ./build_tools/valgrind_test.sh',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB test under TSAN
#
TSAN_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under TSAN',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 check',
'user':'root'
},
],
$REPORT_EMAIL
}
]"
#
# RocksDB crash test under TSAN
#
TSAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb Crash Test under TSAN',
$ONCALL,
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Compile and run',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 crash_test',
'user':'root'
},
],
}
]"
#
# RocksDB format compatible
#
SCRIPT='
export TEST_TMPDIR=/dev/shm/rocksdb
rm -rf /dev/shm/rocksdb
mkdir /dev/shm/rocksdb
echo "
if [ -e "build_tools/build_detect_platform" ]
then
sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
rm -rf build_tools/build_detect_platform
cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
chmod +x build_tools/build_detect_platform
fi
if [ -e "build_detect_platform" ]
then
sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
rm -rf build_detect_platform
cp $TEST_TMPDIR/temp_build_file build_detect_platform
chmod +x build_detect_platform
fi
make ldb -j32
if [ -e "build_detect_platform" ]
then
git checkout -- build_detect_platform
fi
if [ -e "build_tools/build_detect_platform" ]
then
git checkout -- build_tools/build_detect_platform
fi
" > temp_build_ldb.sh
sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
chmod +x tools/temp_check_format_compatible.sh
tools/temp_check_format_compatible.sh
'
FORMAT_COMPATIBLE_COMMANDS="[
{
'name':'Rocksdb Format Compatible tests',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'echo \'$SCRIPT\' > .legocmd && cat .legocmd && bash .legocmd',
'user':'root'
},
],
}
]"
#
# RocksDB no compression
#
SCRIPT='
export TEST_TMPDIR=/dev/shm/rocksdb
rm -rf /dev/shm/rocksdb
mkdir /dev/shm/rocksdb
make clean
cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv
dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
mv .tmp.Makefile Makefile
make $DEBUG J=1 check
'
NO_COMPRESSION_COMMANDS="[
{
'name':'Rocksdb No Compression tests',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'echo \'$SCRIPT\' > .legocmd && cat .legocmd && bash .legocmd',
'user':'root'
},
],
}
]"
#
# RocksDB regression
#
SCRIPT='
time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
# ======= report size to ODS ========
# parameters: $1 -- key, $2 -- value
function send_size_to_ods {
curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
--connect-timeout 60
}
# === normal build ===
make clean
make -j$(nproc) static_lib
send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
strip librocksdb.a
send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
make -j$(nproc) shared_lib
send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
strip `readlink -f librocksdb.so`
send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
# === lite build ===
make clean
OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
strip librocksdb.a
send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
strip `readlink -f librocksdb.so`
send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
'
REGRESSION_COMMANDS="[
{
'name':'Rocksdb regression commands',
$ONCALL,
'steps': [
$CLEANUP_ENV,
{
'name':'Make and run script',
'shell':'echo \'$SCRIPT\' > .legocmd && cat .legocmd && bash .legocmd',
'user':'root'
},
],
}
]"
case $1 in
punit)
echo $PARALLEL_UNIT_TEST_COMMANDS
;;
unit)
echo $UNIT_TEST_COMMANDS
;;
unit_481)
echo $UNIT_TEST_COMMANDS_481
;;
clang_unit)
echo $CLANG_UNIT_TEST_COMMANDS
;;
clang_analyze)
echo $CLANG_ANALYZE_COMMANDS
;;
code_cov)
echo $CODE_COV_COMMANDS
;;
unity)
echo $UNITY_COMMANDS
;;
lite)
echo $LITE_BUILD_COMMANDS
;;
stress_crash)
echo $STRESS_CRASH_TEST_COMMANDS
;;
asan)
echo $ASAN_TEST_COMMANDS
;;
asan_crash)
echo $ASAN_CRASH_TEST_COMMANDS
;;
valgrind)
echo $VALGRIND_TEST_COMMANDS
;;
tsan)
echo $TSAN_UNIT_TEST_COMMANDS
;;
tsan_crash)
echo $TSAN_CRASH_TEST_COMMANDS
;;
format_compatible)
echo $FORMAT_COMPATIBLE_COMMANDS
;;
no_compression)
echo $NO_COMPRESSION_COMMANDS
;;
regression)
echo $REGRESSION_COMMANDS
;;
*)
echo "Invalid determinator command"
;;
esac