#!/usr/bin/env bash # This script is executed by Sandcastle # to determine next steps to run # Usage: # EMAIL= ONCALL= TRIGGER= SUBSCRIBER= WORKINGDIR= rocksdb-lego-determinator # # Input Value # ------------------------------------------------------------------------- # EMAIL Email address to report on trigger conditions # ONCALL Email address to raise a task on failure # TRIGGER Trigger conditions for email. Valid values are fail, warn, all # SUBSCRIBER Email addresss to add as subscriber for task # WORKINGDIR Working directory # # Report configuration # REPORT_EMAIL= if [ ! -z $EMAIL ]; then if [ -z $TRIGGER ]; then TRIGGER="fail" fi REPORT_EMAIL=" { \"type\":\"email\", \"triggers\": [ \"$TRIGGER\" ], \"emails\":[\"$EMAIL\"] }" fi CREATE_TASK= if [ ! -z $ONCALL ]; then CREATE_TASK=" { \"type\":\"task\", \"triggers\":[ \"fail\" ], \"priority\":0, \"subscribers\":[ \"$SUBSCRIBER\" ], \"tags\":[ \"rocksdb\", \"ci\" ] }" fi # For now, create the tasks using only the dedicated task creation tool. CREATE_TASK= REPORT= if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then REPORT=",\"report\": [ $REPORT_EMAIL, $CREATE_TASK ]" fi # Working directory for the following command, default to current directory WORKING_DIR=. if [ ! -z $WORKINGDIR ]; then WORKING_DIR=$WORKINGDIR fi # # Helper variables # CLEANUP_ENV=" { \"name\":\"Cleanup environment\", \"shell\":\"cd $WORKING_DIR; rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean\", \"user\":\"root\" }" UPLOAD_DB_DIR=" { \"name\":\"Upload database directory\", \"shell\":\"tar -cvzf rocksdb_db.tar.gz /dev/shm/rocksdb/\", \"user\":\"root\", \"cleanup\":true, \"provide_artifacts\": [ { \"name\":\"rocksdb_db_dir\", \"paths\": [\"rocksdb_db.tar.gz\"], \"bundle\": false } ] }" # set default RATIO to 1, which sets J=$(nproc) and j=$(nproc) if [ -z $RATIO ]; then RATIO=1 fi # Should probably be called PARALLEL_TEST if [ -z $PARALLEL_J ]; then PARALLEL_J="J=$(expr $(nproc) / ${RATIO})" fi # Should probably be called PARALLEL_MAKE if [ -z $PARALLEL_j ]; then PARALLEL_j="-j$(expr $(nproc) / ${RATIO})" fi PARALLELISM="$PARALLEL_J $PARALLEL_j" DEBUG="OPT=-g" SHM="TEST_TMPDIR=/dev/shm/rocksdb" NON_SHM="TMPD=/tmp/rocksdb_test_tmp" GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" ASAN="COMPILE_WITH_ASAN=1" CLANG="USE_CLANG=1" TSAN="COMPILE_WITH_TSAN=1" UBSAN="COMPILE_WITH_UBSAN=1" ASAN_CRASH="ASAN_OPTIONS=disable_coredump=0" TSAN_CRASH="CRASH_TEST_EXT_ARGS=\\\"--compression_type=zstd --log2_keys_per_lock=22\\\"" NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd" DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH" PARSER="\"parser\":\"/usr/bin/env python3 build_tools/error_filter.py $1\"" CONTRUN_NAME="ROCKSDB_CONTRUN_NAME" SKIP_FORMAT_CHECKS="SKIP_FORMAT_BUCK_CHECKS=1" # This code is getting called under various scenarios. What we care about is to # understand when it's called from nightly contruns because in that case we'll # create tasks for any failures. To follow the existing pattern, we'll check # the value of $ONCALL. If it's a diff then just call `false` to make sure # that errors will be properly propagated to the caller. if [ ! -z $ONCALL ]; then TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL" else TASK_CREATION_TOOL="false" fi # # A mechanism to disable tests temporarily # DISABLE_COMMANDS="[ { \"name\":\"Disable test\", \"oncall\":\"$ONCALL\", \"steps\": [ { \"name\":\"Job disabled. Please contact test owner\", \"shell\":\"exit 1\", \"user\":\"root\" } ] } ]" # # RocksDB unit test # UNIT_TEST_COMMANDS="[ { \"name\":\"Rocksdb Unit Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and test RocksDB debug version\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB unit test not under /dev/shm # UNIT_TEST_NON_SHM_COMMANDS="[ { \"name\":\"Rocksdb Unit Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and test RocksDB debug version\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $NON_SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB release build and unit tests # RELEASE_BUILD_COMMANDS="[ { \"name\":\"Rocksdb Release Build\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build RocksDB release\", \"shell\":\"cd $WORKING_DIR; make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB unit test on gcc-4.8.1 # UNIT_TEST_COMMANDS_481="[ { \"name\":\"Rocksdb Unit Test on GCC 4.8.1\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and test RocksDB debug version\", \"shell\":\"cd $WORKING_DIR; $SHM $GCC_481 $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB release build and unit tests # RELEASE_BUILD_COMMANDS_481="[ { \"name\":\"Rocksdb Release on GCC 4.8.1\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build RocksDB release on GCC 4.8.1\", \"shell\":\"cd $WORKING_DIR; $GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB unit test with CLANG # CLANG_UNIT_TEST_COMMANDS="[ { \"name\":\"Rocksdb Unit Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and test RocksDB debug\", \"shell\":\"cd $WORKING_DIR; $CLANG $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB release build with CLANG # CLANG_RELEASE_BUILD_COMMANDS="[ { \"name\":\"Rocksdb CLANG Release Build\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build RocksDB release\", \"shell\":\"cd $WORKING_DIR; $CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB analyze # CLANG_ANALYZE_COMMANDS="[ { \"name\":\"Rocksdb analyze\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"RocksDB build and analyze\", \"shell\":\"cd $WORKING_DIR; $CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB code coverage # CODE_COV_COMMANDS="[ { \"name\":\"Rocksdb Unit Test Code Coverage\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build, test and collect code coverage info\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB unity # UNITY_COMMANDS="[ { \"name\":\"Rocksdb Unity\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build, test unity test\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG V=1 make $PARALLELISM unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # Build RocksDB lite # LITE_BUILD_COMMANDS="[ { \"name\":\"Rocksdb Lite build\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build RocksDB debug version\", \"shell\":\"cd $WORKING_DIR; $SKIP_FORMAT_CHECKS make $PARALLELISM LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB stress/crash test # STRESS_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb Stress and Crash Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug stress tests\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Build and run RocksDB debug crash tests\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB blackbox stress/crash test # BLACKBOX_STRESS_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb Blackbox Stress and Crash Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug stress tests\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Build and run RocksDB debug blackbox crash tests\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM blackbox_crash_test || $CONTRUN_NAME=blackbox_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB whitebox stress/crash test # WHITEBOX_STRESS_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb Whitebox Stress and Crash Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug stress tests\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Build and run RocksDB debug whitebox crash tests\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM whitebox_crash_test || $CONTRUN_NAME=whitebox_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB stress/crash test with atomic flush # STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ { \"name\":\"Rocksdb Stress and Crash Test with atomic flush\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug stress tests\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Build and run RocksDB debug crash tests with atomic flush\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB stress/crash test with txn # STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[ { \"name\":\"Rocksdb Stress and Crash Test with txn\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug stress tests\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Build and run RocksDB debug crash tests with txn\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB stress/crash test with timestamp # STRESS_CRASH_TEST_WITH_TS_COMMANDS="[ { \"name\":\"Rocksdb Stress and Crash Test with ts\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug stress tests\", \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Build and run RocksDB debug crash tests with ts\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_ts || $CONTRUN_NAME=crash_test_with_ts $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # RocksDB write stress test. # We run on disk device on purpose (i.e. no $SHM) # because we want to add some randomness to fsync commands WRITE_STRESS_COMMANDS="[ { \"name\":\"Rocksdb Write Stress Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB write stress tests\", \"shell\":\"cd $WORKING_DIR; make write_stress && /usr/bin/env python3 tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ], \"artifacts\": [{\"name\": \"database\", \"paths\": [\"/tmp/rocksdb_write_stress\"]}] $REPORT } ]" # # RocksDB test under address sanitizer # ASAN_TEST_COMMANDS="[ { \"name\":\"Rocksdb Unit Test under ASAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Test RocksDB debug under ASAN\", \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $ASAN $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB crash testing under address sanitizer # ASAN_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb crash test under ASAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug asan_crash_test\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB blackbox crash testing under address sanitizer # ASAN_BLACKBOX_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb blackbox crash test under ASAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug blackbox asan_crash_test\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM blackbox_asan_crash_test || $CONTRUN_NAME=blackbox_asan_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB whitebox crash testing under address sanitizer # ASAN_WHITEBOX_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb whitebox crash test under ASAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug whitebox asan_crash_test\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM whitebox_asan_crash_test || $CONTRUN_NAME=whitebox_asan_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash testing with atomic flush under address sanitizer # ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ { \"name\":\"Rocksdb crash test with atomic flush under ASAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug asan_crash_test_with_atomic_flush\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash testing with txn under address sanitizer # ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[ { \"name\":\"Rocksdb crash test with txn under ASAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug asan_crash_test_with_txn\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB test under undefined behavior sanitizer # UBSAN_TEST_COMMANDS="[ { \"name\":\"Rocksdb Unit Test under UBSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Test RocksDB debug under UBSAN\", \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $UBSAN $CLANG $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB crash testing under undefined behavior sanitizer # UBSAN_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb crash test under UBSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug ubsan_crash_test\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash testing under undefined behavior sanitizer # UBSAN_BLACKBOX_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb blackbox crash test under UBSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug blackbox ubsan_crash_test\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM blackbox_ubsan_crash_test || $CONTRUN_NAME=blackbox_ubsan_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash testing under undefined behavior sanitizer # UBSAN_WHITEBOX_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb whitebox crash test under UBSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug whitebox ubsan_crash_test\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM whitebox_ubsan_crash_test || $CONTRUN_NAME=whitebox_ubsan_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash testing with atomic flush under undefined behavior sanitizer # UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ { \"name\":\"Rocksdb crash test with atomic flush under UBSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug ubsan_crash_test_with_atomic_flush\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash testing with txn under undefined behavior sanitizer # UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ { \"name\":\"Rocksdb crash test with txn under UBSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build and run RocksDB debug ubsan_crash_test_with_txn\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB unit test under valgrind # VALGRIND_TEST_COMMANDS="[ { \"name\":\"Rocksdb Unit Test under valgrind\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Run RocksDB debug unit tests\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB test under TSAN # TSAN_UNIT_TEST_COMMANDS="[ { \"name\":\"Rocksdb Unit Test under TSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Run RocksDB debug unit test\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB crash test under TSAN # TSAN_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb Crash Test under TSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Compile and run\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB blackbox crash test under TSAN # TSAN_BLACKBOX_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb Blackbox Crash Test under TSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Compile and run\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM blackbox_crash_test || $CONTRUN_NAME=tsan_blackbox_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB whitebox crash test under TSAN # TSAN_WHITEBOX_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb Whitebox Crash Test under TSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Compile and run\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM whitebox_crash_test || $CONTRUN_NAME=tsan_whitebox_crash_test $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash test with atomic flush under TSAN # TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ { \"name\":\"Rocksdb Crash Test with atomic flush under TSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Compile and run\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB crash test with txn under TSAN # TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ { \"name\":\"Rocksdb Crash Test with txn under TSAN\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ $CLEANUP_ENV, { \"name\":\"Compile and run\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, $UPLOAD_DB_DIR ] $REPORT } ]" # # RocksDB format compatible # run_format_compatible() { export TEST_TMPDIR=/dev/shm/rocksdb rm -rf /dev/shm/rocksdb mkdir /dev/shm/rocksdb export https_proxy="fwdproxy:8080" tools/check_format_compatible.sh } FORMAT_COMPATIBLE_COMMANDS="[ { \"name\":\"Rocksdb Format Compatible tests\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Run RocksDB debug unit test\", \"shell\":\"cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB no compression # run_no_compression() { export TEST_TMPDIR=/dev/shm/rocksdb rm -rf /dev/shm/rocksdb mkdir /dev/shm/rocksdb make clean cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile mv .tmp.Makefile Makefile export $SKIP_FORMAT_CHECKS make $DEBUG $PARALLELISM check } NO_COMPRESSION_COMMANDS="[ { \"name\":\"Rocksdb No Compression tests\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Run RocksDB debug unit test\", \"shell\":\"cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB regression # run_regression() { time bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKING_DIR/rocksdb.XXXX) $(mktemp rocksdb_test_stats.XXXX) # ======= report size to ODS ======== # parameters: $1 -- key, $2 -- value function send_size_to_ods { curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ --connect-timeout 60 } # === normal build === make clean make -j$(nproc) static_lib send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a) strip librocksdb.a send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a) make clean make -j$(nproc) shared_lib send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`) strip `readlink -f librocksdb.so` send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) # === lite build === make clean make LITE=1 -j$(nproc) static_lib send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a) strip librocksdb.a send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a) make clean make LITE=1 -j$(nproc) shared_lib send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`) strip `readlink -f librocksdb.so` send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) } REGRESSION_COMMANDS="[ { \"name\":\"Rocksdb regression commands\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Make and run script\", \"shell\":\"cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB Java build # JAVA_BUILD_TEST_COMMANDS="[ { \"name\":\"Rocksdb Java Build\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"steps\": [ $CLEANUP_ENV, { \"name\":\"Build RocksDB for Java\", \"shell\":\"cd $WORKING_DIR; $SETUP_JAVA_ENV; $SHM make $PARALLELISM rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" # # RocksDB fbcode stress/crash test # FBCODE_STRESS_CRASH_TEST_COMMANDS="[ { \"name\":\"Rocksdb Fbcode Stress and Crash Test\", \"oncall\":\"$ONCALL\", \"executeLocal\": \"true\", \"timeout\": 86400, \"steps\": [ { \"name\":\"Copy RocksDB code to fbcode repo\", \"shell\":\"cd internal_repo_rocksdb/repo && git init -b main && git add * && git commit -a -m \\\"Make internal_repo_rocksdb/repo a git repo\\\" && cd ../.. && echo Y | python3 rocks/release_script/release_to_fbcode.py -u internal_repo_rocksdb/repo main || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Build RocksDB fbcode stress tests\", \"shell\":\"cd $WORKING_DIR; buck build @mode/dbg rocks/tools:rocks_db_stress || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER }, { \"name\":\"Run RocksDB whitebox crash tests\", \"timeout\": 86400, \"shell\":\"cd $WORKING_DIR; mkdir /dev/shm/rocksdb_fbcode_crash_test && TEST_TMPDIR=\$(mktemp -d --tmpdir=/dev/shm/rocksdb_fbcode_crash_test) python3 rocksdb/src/tools/db_crashtest.py --stress_cmd=buck-out/dbg/gen/rocks/tools/rocks_db_stress -secondary_cache_uri=\\\"$SECONDARY_CACHE_URI\\\" --env_uri=$ENV_URI $EXTRA_DB_STRESS_ARGS -logtostderr=false $TEST_TYPE || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL\", \"user\":\"root\", $PARSER } ] $REPORT } ]" case $1 in unit) echo $UNIT_TEST_COMMANDS ;; unit_non_shm) echo $UNIT_TEST_NON_SHM_COMMANDS ;; release) echo $RELEASE_BUILD_COMMANDS ;; unit_481) echo $UNIT_TEST_COMMANDS_481 ;; release_481) echo $RELEASE_BUILD_COMMANDS_481 ;; clang_unit) echo $CLANG_UNIT_TEST_COMMANDS ;; clang_release) echo $CLANG_RELEASE_BUILD_COMMANDS ;; clang_analyze) echo $CLANG_ANALYZE_COMMANDS ;; code_cov) echo $CODE_COV_COMMANDS ;; unity) echo $UNITY_COMMANDS ;; lite) echo $LITE_BUILD_COMMANDS ;; stress_crash) echo $STRESS_CRASH_TEST_COMMANDS ;; blackbox_stress_crash) echo $BLACKBOX_STRESS_CRASH_TEST_COMMANDS ;; whitebox_stress_crash) echo $WHITEBOX_STRESS_CRASH_TEST_COMMANDS ;; stress_crash_with_atomic_flush) echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS ;; stress_crash_with_txn) echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS ;; stress_crash_with_ts) echo $STRESS_CRASH_TEST_WITH_TS_COMMANDS ;; write_stress) echo $WRITE_STRESS_COMMANDS ;; asan) echo $ASAN_TEST_COMMANDS ;; asan_crash) echo $ASAN_CRASH_TEST_COMMANDS ;; blackbox_asan_crash) echo $ASAN_BLACKBOX_CRASH_TEST_COMMANDS ;; whitebox_asan_crash) echo $ASAN_WHITEBOX_CRASH_TEST_COMMANDS ;; asan_crash_with_atomic_flush) echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS ;; asan_crash_with_txn) echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS ;; ubsan) echo $UBSAN_TEST_COMMANDS ;; ubsan_crash) echo $UBSAN_CRASH_TEST_COMMANDS ;; blackbox_ubsan_crash) echo $UBSAN_BLACKBOX_CRASH_TEST_COMMANDS ;; whitebox_ubsan_crash) echo $UBSAN_WHITEBOX_CRASH_TEST_COMMANDS ;; ubsan_crash_with_atomic_flush) echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS ;; ubsan_crash_with_txn) echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS ;; valgrind) echo $VALGRIND_TEST_COMMANDS ;; tsan) echo $TSAN_UNIT_TEST_COMMANDS ;; tsan_crash) echo $TSAN_CRASH_TEST_COMMANDS ;; blackbox_tsan_crash) echo $TSAN_BLACKBOX_CRASH_TEST_COMMANDS ;; whitebox_tsan_crash) echo $TSAN_WHITEBOX_CRASH_TEST_COMMANDS ;; tsan_crash_with_atomic_flush) echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS ;; tsan_crash_with_txn) echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS ;; format_compatible) echo $FORMAT_COMPATIBLE_COMMANDS ;; run_format_compatible) run_format_compatible ;; no_compression) echo $NO_COMPRESSION_COMMANDS ;; run_no_compression) run_no_compression ;; regression) echo $REGRESSION_COMMANDS ;; run_regression) set -e run_regression set +e ;; java_build) echo $JAVA_BUILD_TEST_COMMANDS ;; fbcode_stress_crash) set -f echo $FBCODE_STRESS_CRASH_TEST_COMMANDS set +f ;; *) echo "Invalid determinator command" exit 1 ;; esac