|
|
|
#!/usr/bin/env bash
|
|
|
|
# This script is executed by Sandcastle
|
|
|
|
# to determine next steps to run
|
|
|
|
|
|
|
|
# Usage:
|
|
|
|
# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> WORKINGDIR=<working_dir> rocksdb-lego-determinator <test-name>
|
|
|
|
#
|
|
|
|
# Input Value
|
|
|
|
# -------------------------------------------------------------------------
|
|
|
|
# EMAIL Email address to report on trigger conditions
|
|
|
|
# ONCALL Email address to raise a task on failure
|
|
|
|
# TRIGGER Trigger conditions for email. Valid values are fail, warn, all
|
|
|
|
# SUBSCRIBER Email addresss to add as subscriber for task
|
|
|
|
# WORKINGDIR Working directory
|
|
|
|
|
|
|
|
#
|
|
|
|
# Report configuration
|
|
|
|
#
|
|
|
|
REPORT_EMAIL=
|
|
|
|
if [ ! -z $EMAIL ]; then
|
|
|
|
if [ -z $TRIGGER ]; then
|
|
|
|
TRIGGER="fail"
|
|
|
|
fi
|
|
|
|
|
|
|
|
REPORT_EMAIL="
|
|
|
|
{
|
|
|
|
'type':'email',
|
|
|
|
'triggers': [ '$TRIGGER' ],
|
|
|
|
'emails':['$EMAIL']
|
|
|
|
},"
|
|
|
|
fi
|
|
|
|
|
|
|
|
CREATE_TASK=
|
|
|
|
if [ ! -z $ONCALL ]; then
|
|
|
|
CREATE_TASK="
|
|
|
|
{
|
|
|
|
'type':'task',
|
|
|
|
'triggers':[ 'fail' ],
|
|
|
|
'priority':0,
|
|
|
|
'subscribers':[ '$SUBSCRIBER' ],
|
|
|
|
'tags':[ 'rocksdb', 'ci' ],
|
|
|
|
},"
|
|
|
|
fi
|
|
|
|
|
|
|
|
# For now, create the tasks using only the dedicated task creation tool.
|
|
|
|
CREATE_TASK=
|
|
|
|
|
|
|
|
REPORT=
|
|
|
|
if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
|
|
|
|
REPORT="'report': [
|
|
|
|
$REPORT_EMAIL
|
|
|
|
$CREATE_TASK
|
|
|
|
]"
|
|
|
|
fi
|
|
|
|
|
|
|
|
# Working directory for the following command, default to current directory
|
|
|
|
WORKING_DIR=.
|
|
|
|
if [ ! -z $WORKINGDIR ]; then
|
|
|
|
WORKING_DIR=$WORKINGDIR
|
|
|
|
fi
|
|
|
|
|
|
|
|
#
|
|
|
|
# Helper variables
|
|
|
|
#
|
|
|
|
CLEANUP_ENV="
|
|
|
|
{
|
|
|
|
'name':'Cleanup environment',
|
|
|
|
'shell':'cd $WORKING_DIR; rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean',
|
|
|
|
'user':'root'
|
|
|
|
}"
|
|
|
|
|
|
|
|
UPLOAD_DB_DIR="
|
|
|
|
{
|
|
|
|
'name':'Upload database directory',
|
|
|
|
'shell':'tar -cvzf rocksdb_db.tar.gz /dev/shm/rocksdb/',
|
|
|
|
'user':'root',
|
|
|
|
'cleanup':true,
|
|
|
|
'provide_artifacts': [
|
|
|
|
{
|
|
|
|
'name':'rocksdb_db_dir',
|
|
|
|
'paths': ['rocksdb_db.tar.gz'],
|
|
|
|
'bundle': false,
|
|
|
|
},
|
|
|
|
],
|
|
|
|
}"
|
|
|
|
|
|
|
|
# We will eventually set the RATIO to 1, but we want do this
|
|
|
|
# in steps. RATIO=$(nproc) will make it work as J=1
|
|
|
|
if [ -z $RATIO ]; then
|
|
|
|
RATIO=$(nproc)
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ -z $PARALLEL_J ]; then
|
|
|
|
PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ -z $PARALLEL_j ]; then
|
|
|
|
PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
|
|
|
|
fi
|
|
|
|
|
|
|
|
PARALLELISM="$PARALLEL_J $PARALLEL_j"
|
|
|
|
|
|
|
|
DEBUG="OPT=-g"
|
|
|
|
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
|
|
|
|
NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
|
|
|
|
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
|
|
|
|
ASAN="COMPILE_WITH_ASAN=1"
|
|
|
|
CLANG="USE_CLANG=1"
|
|
|
|
TSAN="COMPILE_WITH_TSAN=1"
|
|
|
|
UBSAN="COMPILE_WITH_UBSAN=1"
|
|
|
|
TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"'
|
|
|
|
NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd"
|
|
|
|
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
|
|
|
|
HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
|
|
|
|
SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
|
|
|
|
PARSER="'parser':'/usr/bin/env python3 build_tools/error_filter.py $1'"
|
|
|
|
|
|
|
|
CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
|
|
|
|
SKIP_FORMAT_CHECKS="SKIP_FORMAT_BUCK_CHECKS=1"
|
|
|
|
|
|
|
|
# This code is getting called under various scenarios. What we care about is to
|
|
|
|
# understand when it's called from nightly contruns because in that case we'll
|
|
|
|
# create tasks for any failures. To follow the existing pattern, we'll check
|
|
|
|
# the value of $ONCALL. If it's a diff then just call `false` to make sure
|
|
|
|
# that errors will be properly propagated to the caller.
|
|
|
|
if [ ! -z $ONCALL ]; then
|
|
|
|
TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL"
|
|
|
|
else
|
|
|
|
TASK_CREATION_TOOL="false"
|
|
|
|
fi
|
|
|
|
|
|
|
|
#
|
|
|
|
# A mechanism to disable tests temporarily
|
|
|
|
#
|
|
|
|
DISABLE_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Disable test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'steps': [
|
|
|
|
{
|
|
|
|
'name':'Job disabled. Please contact test owner',
|
|
|
|
'shell':'exit 1',
|
|
|
|
'user':'root'
|
|
|
|
},
|
|
|
|
],
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB unit test
|
|
|
|
#
|
|
|
|
UNIT_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and test RocksDB debug version',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB unit test not under /dev/shm
|
|
|
|
#
|
|
|
|
UNIT_TEST_NON_SHM_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and test RocksDB debug version',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $NON_SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB release build and unit tests
|
|
|
|
#
|
|
|
|
RELEASE_BUILD_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Release Build',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build RocksDB release',
|
|
|
|
'shell':'cd $WORKING_DIR; make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB unit test on gcc-4.8.1
|
|
|
|
#
|
|
|
|
UNIT_TEST_COMMANDS_481="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test on GCC 4.8.1',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and test RocksDB debug version',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $GCC_481 $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB release build and unit tests
|
|
|
|
#
|
|
|
|
RELEASE_BUILD_COMMANDS_481="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Release on GCC 4.8.1',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build RocksDB release on GCC 4.8.1',
|
|
|
|
'shell':'cd $WORKING_DIR; $GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB unit test with CLANG
|
|
|
|
#
|
|
|
|
CLANG_UNIT_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and test RocksDB debug',
|
|
|
|
'shell':'cd $WORKING_DIR; $CLANG $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB release build with CLANG
|
|
|
|
#
|
|
|
|
CLANG_RELEASE_BUILD_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb CLANG Release Build',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build RocksDB release',
|
|
|
|
'shell':'cd $WORKING_DIR; $CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB analyze
|
|
|
|
#
|
|
|
|
CLANG_ANALYZE_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb analyze',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'RocksDB build and analyze',
|
|
|
|
'shell':'cd $WORKING_DIR; $CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB code coverage
|
|
|
|
#
|
|
|
|
CODE_COV_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test Code Coverage',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build, test and collect code coverage info',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB unity
|
|
|
|
#
|
|
|
|
UNITY_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unity',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build, test unity test',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# Build RocksDB lite
|
|
|
|
#
|
|
|
|
LITE_BUILD_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Lite build',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build RocksDB debug version',
|
|
|
|
'shell':'cd $WORKING_DIR; $SKIP_FORMAT_CHECKS make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB stress/crash test
|
|
|
|
#
|
|
|
|
STRESS_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Stress and Crash Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug crash tests',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB blackbox stress/crash test
|
|
|
|
#
|
|
|
|
BLACKBOX_STRESS_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Blackbox Stress and Crash Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug blackbox crash tests',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 blackbox_crash_test || $CONTRUN_NAME=blackbox_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB whitebox stress/crash test
|
|
|
|
#
|
|
|
|
WHITEBOX_STRESS_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Whitebox Stress and Crash Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug whitebox crash tests',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 whitebox_crash_test || $CONTRUN_NAME=whitebox_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB stress/crash test with atomic flush
|
|
|
|
#
|
|
|
|
STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Stress and Crash Test with atomic flush',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug crash tests with atomic flush',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB stress/crash test with txn
|
|
|
|
#
|
|
|
|
STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Stress and Crash Test with txn',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug crash tests with txn',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB stress/crash test with timestamp
|
|
|
|
#
|
|
|
|
STRESS_CRASH_TEST_WITH_TS_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Stress and Crash Test with ts',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug crash tests with ts',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_ts || $CONTRUN_NAME=crash_test_with_ts $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
# RocksDB write stress test.
|
|
|
|
# We run on disk device on purpose (i.e. no $SHM)
|
|
|
|
# because we want to add some randomness to fsync commands
|
|
|
|
WRITE_STRESS_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Write Stress Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB write stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; make write_stress && /usr/bin/env python3 tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
}
|
|
|
|
],
|
|
|
|
'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB test under address sanitizer
|
|
|
|
#
|
|
|
|
ASAN_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test under ASAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Test RocksDB debug under ASAN',
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $ASAN $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
}
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing under address sanitizer
|
|
|
|
#
|
|
|
|
ASAN_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb crash test under ASAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug asan_crash_test',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB blackbox crash testing under address sanitizer
|
|
|
|
#
|
|
|
|
ASAN_BLACKBOX_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb blackbox crash test under ASAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug blackbox asan_crash_test',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 blackbox_asan_crash_test || $CONTRUN_NAME=blackbox_asan_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB whitebox crash testing under address sanitizer
|
|
|
|
#
|
|
|
|
ASAN_WHITEBOX_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb whitebox crash test under ASAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug whitebox asan_crash_test',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 whitebox_asan_crash_test || $CONTRUN_NAME=whitebox_asan_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing with atomic flush under address sanitizer
|
|
|
|
#
|
|
|
|
ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb crash test with atomic flush under ASAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing with txn under address sanitizer
|
|
|
|
#
|
|
|
|
ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb crash test with txn under ASAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug asan_crash_test_with_txn',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB test under undefined behavior sanitizer
|
|
|
|
#
|
|
|
|
UBSAN_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test under UBSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Test RocksDB debug under UBSAN',
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $UBSAN $CLANG $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
}
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing under undefined behavior sanitizer
|
|
|
|
#
|
|
|
|
UBSAN_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb crash test under UBSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug ubsan_crash_test',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing under undefined behavior sanitizer
|
|
|
|
#
|
|
|
|
UBSAN_BLACKBOX_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb blackbox crash test under UBSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug blackbox ubsan_crash_test',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 blackbox_ubsan_crash_test || $CONTRUN_NAME=blackbox_ubsan_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing under undefined behavior sanitizer
|
|
|
|
#
|
|
|
|
UBSAN_WHITEBOX_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb whitebox crash test under UBSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug whitebox ubsan_crash_test',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 whitebox_ubsan_crash_test || $CONTRUN_NAME=whitebox_ubsan_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing with atomic flush under undefined behavior sanitizer
|
|
|
|
#
|
|
|
|
UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb crash test with atomic flush under UBSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash testing with txn under undefined behavior sanitizer
|
|
|
|
#
|
|
|
|
UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb crash test with txn under UBSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build and run RocksDB debug ubsan_crash_test_with_txn',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB unit test under valgrind
|
|
|
|
#
|
|
|
|
VALGRIND_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test under valgrind',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Run RocksDB debug unit tests',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; $SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB test under TSAN
|
|
|
|
#
|
|
|
|
TSAN_UNIT_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Unit Test under TSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Run RocksDB debug unit test',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash test under TSAN
|
|
|
|
#
|
|
|
|
TSAN_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Crash Test under TSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Compile and run',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB blackbox crash test under TSAN
|
|
|
|
#
|
|
|
|
TSAN_BLACKBOX_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Blackbox Crash Test under TSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Compile and run',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 blackbox_crash_test || $CONTRUN_NAME=tsan_blackbox_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB whitebox crash test under TSAN
|
|
|
|
#
|
|
|
|
TSAN_WHITEBOX_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Whitebox Crash Test under TSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Compile and run',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 whitebox_crash_test || $CONTRUN_NAME=tsan_whitebox_crash_test $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash test with atomic flush under TSAN
|
|
|
|
#
|
|
|
|
TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Crash Test with atomic flush under TSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Compile and run',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB crash test with txn under TSAN
|
|
|
|
#
|
|
|
|
TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Crash Test with txn under TSAN',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Compile and run',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
$UPLOAD_DB_DIR,
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB format compatible
|
|
|
|
#
|
|
|
|
|
|
|
|
run_format_compatible()
|
|
|
|
{
|
|
|
|
export TEST_TMPDIR=/dev/shm/rocksdb
|
|
|
|
rm -rf /dev/shm/rocksdb
|
|
|
|
mkdir /dev/shm/rocksdb
|
|
|
|
|
|
|
|
export https_proxy="fwdproxy:8080"
|
|
|
|
|
|
|
|
tools/check_format_compatible.sh
|
|
|
|
}
|
|
|
|
|
|
|
|
FORMAT_COMPATIBLE_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Format Compatible tests',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Run RocksDB debug unit test',
|
|
|
|
'shell':'cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB no compression
|
|
|
|
#
|
|
|
|
run_no_compression()
|
|
|
|
{
|
|
|
|
export TEST_TMPDIR=/dev/shm/rocksdb
|
|
|
|
rm -rf /dev/shm/rocksdb
|
|
|
|
mkdir /dev/shm/rocksdb
|
|
|
|
make clean
|
|
|
|
cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
|
|
|
|
mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
|
|
|
|
cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
|
|
|
|
mv .tmp.Makefile Makefile
|
|
|
|
export $SKIP_FORMAT_CHECKS
|
|
|
|
make $DEBUG J=1 check
|
|
|
|
}
|
|
|
|
|
|
|
|
NO_COMPRESSION_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb No Compression tests',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Run RocksDB debug unit test',
|
|
|
|
'shell':'cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB regression
|
|
|
|
#
|
|
|
|
run_regression()
|
|
|
|
{
|
|
|
|
time bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKING_DIR/rocksdb.XXXX) $(mktemp rocksdb_test_stats.XXXX)
|
|
|
|
|
|
|
|
# ======= report size to ODS ========
|
|
|
|
|
|
|
|
# parameters: $1 -- key, $2 -- value
|
|
|
|
function send_size_to_ods {
|
|
|
|
curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
|
|
|
|
--connect-timeout 60
|
|
|
|
}
|
|
|
|
|
|
|
|
# === normal build ===
|
|
|
|
make clean
|
|
|
|
make -j$(nproc) static_lib
|
|
|
|
send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
|
|
|
|
strip librocksdb.a
|
|
|
|
send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
|
|
|
|
|
|
|
|
make clean
|
|
|
|
make -j$(nproc) shared_lib
|
|
|
|
send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
|
|
strip `readlink -f librocksdb.so`
|
|
|
|
send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
|
|
|
|
|
|
# === lite build ===
|
|
|
|
make clean
|
|
|
|
make LITE=1 -j$(nproc) static_lib
|
|
|
|
send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
|
|
|
|
strip librocksdb.a
|
|
|
|
send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
|
|
|
|
|
|
|
|
make clean
|
|
|
|
make LITE=1 -j$(nproc) shared_lib
|
|
|
|
send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
|
|
strip `readlink -f librocksdb.so`
|
|
|
|
send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
|
|
}
|
|
|
|
|
|
|
|
REGRESSION_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb regression commands',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Make and run script',
|
|
|
|
'shell':'cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB Java build
|
|
|
|
#
|
|
|
|
JAVA_BUILD_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Java Build',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'steps': [
|
|
|
|
$CLEANUP_ENV,
|
|
|
|
{
|
|
|
|
'name':'Build RocksDB for Java',
|
|
|
|
'shell':'cd $WORKING_DIR; $SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
#
|
|
|
|
# RocksDB fbcode stress/crash test
|
|
|
|
#
|
|
|
|
FBCODE_STRESS_CRASH_TEST_COMMANDS="[
|
|
|
|
{
|
|
|
|
'name':'Rocksdb Fbcode Stress and Crash Test',
|
|
|
|
'oncall':'$ONCALL',
|
|
|
|
'executeLocal': 'true',
|
|
|
|
'timeout': 86400,
|
|
|
|
'steps': [
|
|
|
|
{
|
|
|
|
'name':'Copy RocksDB code to fbcode repo',
|
|
|
|
'shell':'cd internal_repo_rocksdb/repo && git init -b main && git add * && git commit -a -m \"Make internal_repo_rocksdb/repo a git repo\" && cd ../.. && echo Y | python3 rocks/release_script/release_to_fbcode.py -u internal_repo_rocksdb/repo main || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Build RocksDB fbcode stress tests',
|
|
|
|
'shell':'cd $WORKING_DIR; buck build @mode/dbg rocks/tools:rocks_db_stress || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
{
|
|
|
|
'name':'Run RocksDB whitebox crash tests',
|
|
|
|
'timeout': 86400,
|
|
|
|
'shell':'cd $WORKING_DIR; mkdir /dev/shm/rocksdb_whitebox_crash_test && TEST_TMPDIR=\$(mktemp -d --tmpdir=/dev/shm/rocksdb_whitebox_crash_test) python3 rocksdb/src/tools/db_crashtest.py --stress_cmd=buck-out/dbg/gen/rocks/tools/rocks_db_stress -secondary_cache_uri=\"$SECONDARY_CACHE_URI\" --env_uri=$ENV_URI -checkpoint_one_in=0 -backup_one_in=0 -cache_size=134217728 -cache_numshardbits=4 $EXTRA_DB_STRESS_ARGS -logtostderr=false whitebox || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL',
|
|
|
|
'user':'root',
|
|
|
|
$PARSER
|
|
|
|
},
|
|
|
|
],
|
|
|
|
$REPORT
|
|
|
|
}
|
|
|
|
]"
|
|
|
|
|
|
|
|
|
|
|
|
case $1 in
|
|
|
|
unit)
|
|
|
|
echo $UNIT_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
unit_non_shm)
|
|
|
|
echo $UNIT_TEST_NON_SHM_COMMANDS
|
|
|
|
;;
|
|
|
|
release)
|
|
|
|
echo $RELEASE_BUILD_COMMANDS
|
|
|
|
;;
|
|
|
|
unit_481)
|
|
|
|
echo $UNIT_TEST_COMMANDS_481
|
|
|
|
;;
|
|
|
|
release_481)
|
|
|
|
echo $RELEASE_BUILD_COMMANDS_481
|
|
|
|
;;
|
|
|
|
clang_unit)
|
|
|
|
echo $CLANG_UNIT_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
clang_release)
|
|
|
|
echo $CLANG_RELEASE_BUILD_COMMANDS
|
|
|
|
;;
|
|
|
|
clang_analyze)
|
|
|
|
echo $CLANG_ANALYZE_COMMANDS
|
|
|
|
;;
|
|
|
|
code_cov)
|
|
|
|
echo $CODE_COV_COMMANDS
|
|
|
|
;;
|
|
|
|
unity)
|
|
|
|
echo $UNITY_COMMANDS
|
|
|
|
;;
|
|
|
|
lite)
|
|
|
|
echo $LITE_BUILD_COMMANDS
|
|
|
|
;;
|
|
|
|
stress_crash)
|
|
|
|
echo $STRESS_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
blackbox_stress_crash)
|
|
|
|
echo $BLACKBOX_STRESS_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
whitebox_stress_crash)
|
|
|
|
echo $WHITEBOX_STRESS_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
stress_crash_with_atomic_flush)
|
|
|
|
echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
|
|
;;
|
|
|
|
stress_crash_with_txn)
|
|
|
|
echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS
|
|
|
|
;;
|
|
|
|
stress_crash_with_ts)
|
|
|
|
echo $STRESS_CRASH_TEST_WITH_TS_COMMANDS
|
|
|
|
;;
|
|
|
|
write_stress)
|
|
|
|
echo $WRITE_STRESS_COMMANDS
|
|
|
|
;;
|
|
|
|
asan)
|
|
|
|
echo $ASAN_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
asan_crash)
|
|
|
|
echo $ASAN_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
blackbox_asan_crash)
|
|
|
|
echo $ASAN_BLACKBOX_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
whitebox_asan_crash)
|
|
|
|
echo $ASAN_WHITEBOX_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
asan_crash_with_atomic_flush)
|
|
|
|
echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
|
|
;;
|
|
|
|
asan_crash_with_txn)
|
|
|
|
echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS
|
|
|
|
;;
|
|
|
|
ubsan)
|
|
|
|
echo $UBSAN_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
ubsan_crash)
|
|
|
|
echo $UBSAN_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
blackbox_ubsan_crash)
|
|
|
|
echo $UBSAN_BLACKBOX_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
whitebox_ubsan_crash)
|
|
|
|
echo $UBSAN_WHITEBOX_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
ubsan_crash_with_atomic_flush)
|
|
|
|
echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
|
|
;;
|
|
|
|
ubsan_crash_with_txn)
|
|
|
|
echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS
|
|
|
|
;;
|
|
|
|
valgrind)
|
|
|
|
echo $VALGRIND_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
tsan)
|
|
|
|
echo $TSAN_UNIT_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
tsan_crash)
|
|
|
|
echo $TSAN_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
blackbox_tsan_crash)
|
|
|
|
echo $TSAN_BLACKBOX_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
whitebox_tsan_crash)
|
|
|
|
echo $TSAN_WHITEBOX_CRASH_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
tsan_crash_with_atomic_flush)
|
|
|
|
echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
|
|
;;
|
|
|
|
tsan_crash_with_txn)
|
|
|
|
echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS
|
|
|
|
;;
|
|
|
|
format_compatible)
|
|
|
|
echo $FORMAT_COMPATIBLE_COMMANDS
|
|
|
|
;;
|
|
|
|
run_format_compatible)
|
|
|
|
run_format_compatible
|
|
|
|
;;
|
|
|
|
no_compression)
|
|
|
|
echo $NO_COMPRESSION_COMMANDS
|
|
|
|
;;
|
|
|
|
run_no_compression)
|
|
|
|
run_no_compression
|
|
|
|
;;
|
|
|
|
regression)
|
|
|
|
echo $REGRESSION_COMMANDS
|
|
|
|
;;
|
|
|
|
run_regression)
|
|
|
|
set -e
|
|
|
|
run_regression
|
|
|
|
set +e
|
|
|
|
;;
|
|
|
|
java_build)
|
|
|
|
echo $JAVA_BUILD_TEST_COMMANDS
|
|
|
|
;;
|
|
|
|
fbcode_stress_crash)
|
|
|
|
set -f
|
|
|
|
echo $FBCODE_STRESS_CRASH_TEST_COMMANDS
|
|
|
|
set +f
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
echo "Invalid determinator command"
|
|
|
|
exit 1
|
|
|
|
;;
|
|
|
|
esac
|