NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Avoid duplicate task creation for RocksDB contruns

Browse Source 
Summary: We rely on a basic evaluation logic: when using an expression `A || B`, the `B` part will only get evaluated when `A` fails. Therefore the task creation tool is guaranteed to run if previous build/test step failed. To indicate the correct return value to shell, the task creation tool will call `exit(1)` which will cause Sandcastle to mark it as a failure.

Test Plan:
- Land the changes.
- Trigger a RocksDB contrun - observe the results.
- Look at the results from the nightly runs and fix issues as necessary.

Current testing so far has been in isolation for various components.

Reviewers: andrewkr, IslamAbdelRahman, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D60909
main
Gunnar Kudrjavets 9 years ago
parent d4c45428af
commit 9c8ac144bd
1 changed files with 45 additions and 29 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 74
      build_tools/rocksdb-lego-determinator

74
build_tools/rocksdb-lego-determinator
Unescape View File

@ -8,7 +8,7 @@
# Input Value
# -------------------------------------------------------------------------
# EMAIL Email address to report on trigger conditions
# ONCAL Email address to raise a task on failure
# ONCALL Email address to raise a task on failure
# TRIGGER Trigger conditions for email. Valid values are fail, warn, all
# SUBSCRIBER Email addresss to add as subscriber for task
#
@ -42,6 +42,9 @@ if [ ! -z $ONCALL ]; then
},"
fi
# For now, create the tasks using only the dedicated task creation tool.
CREATE_TASK=
REPORT=
if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
REPORT="'report': [
@ -89,6 +92,19 @@ HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.
SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-7u10-64/; export PATH=\$PATH:\$JAVA_HOME/bin"
PARSER="'parser':'python build_tools/error_filter.py $1'"
CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
# This code is getting called under various scenarios. What we care about is to
# understand when it's called from nightly contruns because in that case we'll
# create tasks for any failures. To follow the existing pattern, we'll check
# the value of $ONCALL. If it's a diff then just call `false` to make sure
# that errors will be properly propagated to the caller.
if [ ! -z $ONCALL ]; then
TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb"
else
TASK_CREATION_TOOL="false"
fi
ARTIFACTS=" 'artifacts': [
{
'name':'database',
@ -126,14 +142,14 @@ PARALLEL_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check $TASK_CREATION_TOOL || cat t/log-*',
'user':'root',
$PARSER
},
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version under gcc-4.8.1',
'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check_gcc481 $TASK_CREATION_TOOL || cat t/log-*',
'user':'root',
$PARSER
},
@ -153,7 +169,7 @@ UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $DEBUG make $PARALLELISM check',
'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -175,7 +191,7 @@ UNIT_TEST_NON_SHM_COMMANDS="[
{
'name':'Build and test RocksDB debug version',
'timeout': 86400,
'shell':'$DEBUG make $PARALLELISM check',
'shell':'$DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -195,7 +211,7 @@ RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB release',
'shell':'make $PARALLEL_j release',
'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -215,7 +231,7 @@ UNIT_TEST_COMMANDS_481="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check',
'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -235,7 +251,7 @@ RELEASE_BUILD_COMMANDS_481="[
$CLEANUP_ENV,
{
'name':'Build RocksDB release on GCC 4.8.1',
'shell':'$GCC_481 make $PARALLEL_j release',
'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -255,7 +271,7 @@ CLANG_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug',
'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check',
'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -275,7 +291,7 @@ CLANG_RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB release',
'shell':'$CLANG make $PARALLEL_j release',
'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -295,7 +311,7 @@ CLANG_ANALYZE_COMMANDS="[
$CLEANUP_ENV,
{
'name':'RocksDB build and analyze',
'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze',
'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -315,7 +331,7 @@ CODE_COV_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build, test and collect code coverage info',
'shell':'$SHM $DEBUG make $PARALLELISM coverage',
'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -335,7 +351,7 @@ UNITY_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build, test unity test',
'shell':'$SHM $DEBUG V=1 make J=1 unity_test',
'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -355,7 +371,7 @@ LITE_BUILD_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$LITE make J=1 static_lib',
'shell':'$LITE make J=1 static_lib || $CONTRUN_NAME=lite_static_lib $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -375,7 +391,7 @@ LITE_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$SHM $LITE make J=1 check',
'shell':'$SHM $LITE make J=1 check || $CONTRUN_NAME=lite_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -396,14 +412,14 @@ STRESS_CRASH_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug stress tests',
'shell':'$SHM $DEBUG make J=1 db_stress',
'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
{
'name':'Build and run RocksDB debug crash tests',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 crash_test',
'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
@ -424,7 +440,7 @@ WRITE_STRESS_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and run RocksDB write stress tests',
'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress',
'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
@ -446,7 +462,7 @@ ASAN_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under ASAN',
'shell':'set -o pipefail && $SHM $ASAN $DEBUG make $PARALLELISM asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'user':'root',
$PARSER
}
@ -468,7 +484,7 @@ ASAN_CRASH_TEST_COMMANDS="[
{
'name':'Build and run RocksDB debug asan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 asan_crash_test',
'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -488,7 +504,7 @@ UBSAN_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under UBSAN',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
@ -510,7 +526,7 @@ UBSAN_CRASH_TEST_COMMANDS="[
{
'name':'Build and run RocksDB debug ubsan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -532,7 +548,7 @@ VALGRIND_TEST_COMMANDS="[
{
'name':'Run RocksDB debug unit tests',
'timeout': 86400,
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make $PARALLELISM valgrind_check',
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make $PARALLELISM valgrind_check || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -554,7 +570,7 @@ TSAN_UNIT_TEST_COMMANDS="[
{
'name':'Run RocksDB debug unit test',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check',
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -576,7 +592,7 @@ TSAN_CRASH_TEST_COMMANDS="[
{
'name':'Compile and run',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test',
'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -638,7 +654,7 @@ FORMAT_COMPATIBLE_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible',
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -671,7 +687,7 @@ NO_COMPRESSION_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_no_compression',
'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -728,7 +744,7 @@ REGRESSION_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Make and run script',
'shell':'build_tools/rocksdb-lego-determinator run_regression',
'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL'',
'user':'root',
$PARSER
},
@ -748,7 +764,7 @@ JAVA_BUILD_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB for Java',
'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava',
'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},

Write Preview
Loading…
Cancel
Save