diff --git a/Makefile b/Makefile index f161d7d37..46b1b5be9 100644 --- a/Makefile +++ b/Makefile @@ -2372,10 +2372,10 @@ jtest: rocksdbjava jdb_bench: cd java;$(MAKE) db_bench; -commit_prereq: build_tools/rocksdb-lego-determinator \ - build_tools/precommit_checker.py - J=$(J) build_tools/precommit_checker.py unit clang_unit release clang_release tsan asan ubsan lite unit_non_shm - $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava; +commit_prereq: + echo "TODO: bring this back using parts of old precommit_checker.py and rocksdb-lego-determinator" + false # J=$(J) build_tools/precommit_checker.py unit clang_unit release clang_release tsan asan ubsan lite unit_non_shm + # $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava; # --------------------------------------------------------------------------- # Platform-specific compilation diff --git a/build_tools/precommit_checker.py b/build_tools/precommit_checker.py deleted file mode 100755 index 212bb5227..000000000 --- a/build_tools/precommit_checker.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import argparse -import subprocess -import sys -import re -import os -import time - - -# -# Simple logger -# - -class Log: - - def __init__(self, filename): - self.filename = filename - self.f = open(self.filename, 'w+') - - def caption(self, str): - line = "\n##### %s #####\n" % str - if self.f: - self.f.write("%s \n" % line) - else: - print(line) - - def error(self, str): - data = "\n\n##### ERROR ##### %s" % str - if self.f: - self.f.write("%s \n" % data) - else: - print(data) - - def log(self, str): - if self.f: - self.f.write("%s \n" % str) - else: - print(str) - -# -# Shell Environment -# - - -class Env(object): - - def __init__(self, logfile, tests): - self.tests = tests - self.log = Log(logfile) - - def shell(self, cmd, path=os.getcwd()): - if path: - os.chdir(path) - - self.log.log("==== shell session ===========================") - self.log.log("%s> %s" % (path, cmd)) - p = subprocess.run("cd %s; %s" % (path, cmd), shell=True, - capture_output=True, text=True) - self.log.log("status = %s" % p.returncode) - self.log.log("stdout = %s" % p.stdout) - if p.stderr: - self.log.log("stderr = %s" % p.stderr) - self.log.log("============================================== \n\n") - return p.returncode, p.stdout - -# -# Pre-commit checker -# - - -class PreCommitChecker(Env): - - def __init__(self, args): - Env.__init__(self, args.logfile, args.tests) - self.ignore_failure = args.ignore_failure - - # - # Get commands for a given job from the determinator file - # - def get_commands(self, test): - status, out = self.shell( - "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".") - return status, out - - # - # Run a specific CI job - # - def run_test(self, test): - self.log.caption("Running test %s locally" % test) - - # get commands for the CI job determinator - status, cmds = self.get_commands(test) - if status != 0: - self.log.error("Error getting commands for test %s" % test) - return False - - # Parse the JSON to extract the commands to run - cmds = re.findall('"shell":"([^\"]*)', cmds) - - if len(cmds) == 0: - self.log.log("No commands found") - return False - - # Run commands - for cmd in cmds: - # Run the command - status, out = self.shell(cmd, ".") - if status != 0: - self.log.error("Error running command %s for test %s" - % (cmd, test)) - return False - - return True - - # - # Run specified CI jobs - # - def run_tests(self): - if not self.tests: - self.log.error("Invalid args. Please provide tests") - return False - - self.print_separator() - self.print_row("TEST", "RESULT") - self.print_separator() - - result = True - for test in self.tests: - start_time = time.time() - self.print_test(test) - result = self.run_test(test) - elapsed_min = (time.time() - start_time) / 60 - if not result: - self.log.error("Error running test %s" % test) - self.print_result("FAIL (%dm)" % elapsed_min) - if not self.ignore_failure: - return False - result = False - else: - self.print_result("PASS (%dm)" % elapsed_min) - - self.print_separator() - return result - - # - # Print a line - # - def print_separator(self): - print("".ljust(60, "-")) - - # - # Print two colums - # - def print_row(self, c0, c1): - print("%s%s" % (c0.ljust(40), c1.ljust(20))) - - def print_test(self, test): - print(test.ljust(40), end="") - sys.stdout.flush() - - def print_result(self, result): - print(result.ljust(20)) - -# -# Main -# -parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.') - -# --log -parser.add_argument('--logfile', default='/tmp/precommit-check.log', - help='Log file. Default is /tmp/precommit-check.log') -# --ignore_failure -parser.add_argument('--ignore_failure', action='store_true', default=False, - help='Stop when an error occurs') -# -parser.add_argument('tests', nargs='+', - help='CI test(s) to run. e.g: unit punit asan tsan ubsan') - -args = parser.parse_args() -checker = PreCommitChecker(args) - -print("Please follow log %s" % checker.log.filename) - -if not checker.run_tests(): - print("Error running tests. Please check log file %s" - % checker.log.filename) - sys.exit(1) - -sys.exit(0) diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator deleted file mode 100755 index ab01d1d88..000000000 --- a/build_tools/rocksdb-lego-determinator +++ /dev/null @@ -1,1382 +0,0 @@ -#!/usr/bin/env bash -# This script is executed by Sandcastle -# to determine next steps to run - -# Usage: -# EMAIL= ONCALL= TRIGGER= SUBSCRIBER= WORKINGDIR= rocksdb-lego-determinator -# -# Input Value -# ------------------------------------------------------------------------- -# EMAIL Email address to report on trigger conditions -# ONCALL Email address to raise a task on failure -# TRIGGER Trigger conditions for email. Valid values are fail, warn, all -# SUBSCRIBER Email addresss to add as subscriber for task -# WORKINGDIR Working directory - -# -# Report configuration -# -REPORT_EMAIL= -if [ ! -z $EMAIL ]; then - if [ -z $TRIGGER ]; then - TRIGGER="fail" - fi - - REPORT_EMAIL=" - { - \"type\":\"email\", - \"triggers\": [ \"$TRIGGER\" ], - \"emails\":[\"$EMAIL\"] - }" -fi - -CREATE_TASK= -if [ ! -z $ONCALL ]; then - CREATE_TASK=" - { - \"type\":\"task\", - \"triggers\":[ \"fail\" ], - \"priority\":0, - \"subscribers\":[ \"$SUBSCRIBER\" ], - \"tags\":[ \"rocksdb\", \"ci\" ] - }" -fi - -# For now, create the tasks using only the dedicated task creation tool. -CREATE_TASK= - -REPORT= -if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then - REPORT=",\"report\": [ - $REPORT_EMAIL, - $CREATE_TASK - ]" -fi - -# Working directory for the following command, default to current directory -WORKING_DIR=. -if [ ! -z $WORKINGDIR ]; then - WORKING_DIR=$WORKINGDIR -fi - -# -# Helper variables -# -CLEANUP_ENV=" -{ - \"name\":\"Cleanup environment\", - \"shell\":\"cd $WORKING_DIR; rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean\", - \"user\":\"root\" -}" - -UPLOAD_DB_DIR=" -{ - \"name\":\"Upload database directory\", - \"shell\":\"tar -cvzf rocksdb_db.tar.gz /dev/shm/rocksdb/\", - \"user\":\"root\", - \"cleanup\":true, - \"provide_artifacts\": [ - { - \"name\":\"rocksdb_db_dir\", - \"paths\": [\"rocksdb_db.tar.gz\"], - \"bundle\": false - } - ] -}" - -# set default RATIO to 2, which sets J=$(nproc)/2 and j=$(nproc)/2 -if [ -z $RATIO ]; then - RATIO=2 -fi - -# Should probably be called PARALLEL_TEST -if [ -z $PARALLEL_J ]; then - PARALLEL_J="J=$(expr $(nproc) / ${RATIO})" -fi - -# Should probably be called PARALLEL_MAKE -if [ -z $PARALLEL_j ]; then - PARALLEL_j="-j$(expr $(nproc) / ${RATIO})" -fi - -PARALLELISM="$PARALLEL_J $PARALLEL_j" - -DEBUG="OPT=-g" -SHM="TEST_TMPDIR=/dev/shm/rocksdb" -NON_SHM="TMPD=/tmp/rocksdb_test_tmp" -ASAN="COMPILE_WITH_ASAN=1" -ASAN_TEST_OPTIONS="ASAN_OPTIONS=detect_stack_use_after_return=1" -CLANG="USE_CLANG=1" -TSAN="COMPILE_WITH_TSAN=1" -UBSAN="COMPILE_WITH_UBSAN=1" -ASAN_CRASH="ASAN_OPTIONS=disable_coredump=0:detect_stack_use_after_return=1" -TSAN_CRASH="TSAN_OPTIONS=disable_coredump=0 CRASH_TEST_EXT_ARGS=\\\"--compression_type=zstd --log2_keys_per_lock=22\\\"" -NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd" -UBSAN_CRASH="UBSAN_OPTIONS=disable_coredump=0" -DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" -HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" -SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH" -PARSER="\"parser\":\"/usr/bin/env python3 build_tools/error_filter.py $1\"" - -CONTRUN_NAME="ROCKSDB_CONTRUN_NAME" -SKIP_FORMAT_CHECKS="SKIP_FORMAT_BUCK_CHECKS=1" - -# This code is getting called under various scenarios. What we care about is to -# understand when it's called from nightly contruns because in that case we'll -# create tasks for any failures. To follow the existing pattern, we'll check -# the value of $ONCALL. If it's a diff then just call `false` to make sure -# that errors will be properly propagated to the caller. -if [ ! -z $ONCALL ]; then - TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL" -else - TASK_CREATION_TOOL="false" -fi - -# -# A mechanism to disable tests temporarily -# -DISABLE_COMMANDS="[ - { - \"name\":\"Disable test\", - \"oncall\":\"$ONCALL\", - \"steps\": [ - { - \"name\":\"Job disabled. Please contact test owner\", - \"shell\":\"exit 1\", - \"user\":\"root\" - } - ] - } -]" - -# -# RocksDB unit test -# -UNIT_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and test RocksDB debug version\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB unit test not under /dev/shm -# -UNIT_TEST_NON_SHM_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and test RocksDB debug version\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $NON_SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB release build and unit tests -# -RELEASE_BUILD_COMMANDS="[ - { - \"name\":\"Rocksdb Release Build\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build RocksDB release\", - \"shell\":\"cd $WORKING_DIR; make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB unit test with CLANG -# -CLANG_UNIT_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and test RocksDB debug\", - \"shell\":\"cd $WORKING_DIR; $CLANG $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB release build with CLANG -# -CLANG_RELEASE_BUILD_COMMANDS="[ - { - \"name\":\"Rocksdb CLANG Release Build\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build RocksDB release\", - \"shell\":\"cd $WORKING_DIR; $CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB analyze -# -CLANG_ANALYZE_COMMANDS="[ - { - \"name\":\"Rocksdb analyze\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"RocksDB build and analyze\", - \"shell\":\"cd $WORKING_DIR; $CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB code coverage -# -CODE_COV_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test Code Coverage\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build, test and collect code coverage info\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB unity -# -UNITY_COMMANDS="[ - { - \"name\":\"Rocksdb Unity\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build, test unity test\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG V=1 make $PARALLELISM unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# Build RocksDB lite -# -LITE_BUILD_COMMANDS="[ - { - \"name\":\"Rocksdb Lite build\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build RocksDB debug version\", - \"shell\":\"cd $WORKING_DIR; $SKIP_FORMAT_CHECKS make $PARALLELISM LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB stress/crash test -# -STRESS_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Stress and Crash Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug crash tests\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB blackbox stress/crash test -# -BLACKBOX_STRESS_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Blackbox Stress and Crash Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug blackbox crash tests\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM blackbox_crash_test || $CONTRUN_NAME=blackbox_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB whitebox stress/crash test -# -WHITEBOX_STRESS_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Whitebox Stress and Crash Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug whitebox crash tests\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM whitebox_crash_test || $CONTRUN_NAME=whitebox_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB stress/crash test with atomic flush -# -STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ - { - \"name\":\"Rocksdb Stress and Crash Test with atomic flush\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug crash tests with atomic flush\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB stress/crash test with txn -# -STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[ - { - \"name\":\"Rocksdb Stress and Crash Test with txn\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug crash tests with txn\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB stress/crash test with timestamp -# -STRESS_CRASH_TEST_WITH_TS_COMMANDS="[ - { - \"name\":\"Rocksdb Stress and Crash Test with ts\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug crash tests with ts\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_ts || $CONTRUN_NAME=crash_test_with_ts $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB stress/crash test for write-committed multi-ops transactions -# -STRESS_CRASH_TEST_WITH_MULTIOPS_WC_TXN_COMMANDS="[ - { - \"name\":\"Rocksdb Stress Crash Test with multi-ops transactions (wc)\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug crash tests with multi-ops transactions (wc)\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_multiops_wc_txn || $CONTRUN_NAME=crash_test_with_multiops_wc_txn $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB stress/crash test for write-prepared multi-ops transactions -# -STRESS_CRASH_TEST_WITH_MULTIOPS_WP_TXN_COMMANDS="[ - { - \"name\":\"Rocksdb Stress Crash Test with multi-ops transactions (wp)\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug stress tests\", - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build and run RocksDB debug crash tests with multi-ops transactions (wp)\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make $PARALLELISM crash_test_with_multiops_wp_txn || $CONTRUN_NAME=crash_test_with_multiops_wp_txn $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# RocksDB write stress test. -# We run on disk device on purpose (i.e. no $SHM) -# because we want to add some randomness to fsync commands -WRITE_STRESS_COMMANDS="[ - { - \"name\":\"Rocksdb Write Stress Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB write stress tests\", - \"shell\":\"cd $WORKING_DIR; make write_stress && /usr/bin/env python3 tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ], - \"artifacts\": [{\"name\": \"database\", \"paths\": [\"/tmp/rocksdb_write_stress\"]}] - $REPORT - } -]" - - -# -# RocksDB test under address sanitizer -# -ASAN_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test under ASAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Test RocksDB debug under ASAN\", -\"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $ASAN $ASAN_TEST_OPTIONS $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB crash testing under address sanitizer -# -ASAN_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb crash test under ASAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug asan_crash_test\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB blackbox crash testing under address sanitizer -# -ASAN_BLACKBOX_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb blackbox crash test under ASAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug blackbox asan_crash_test\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM blackbox_asan_crash_test || $CONTRUN_NAME=blackbox_asan_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB whitebox crash testing under address sanitizer -# -ASAN_WHITEBOX_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb whitebox crash test under ASAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug whitebox asan_crash_test\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM whitebox_asan_crash_test || $CONTRUN_NAME=whitebox_asan_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash testing with atomic flush under address sanitizer -# -ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ - { - \"name\":\"Rocksdb crash test with atomic flush under ASAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug asan_crash_test_with_atomic_flush\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash testing with txn under address sanitizer -# -ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[ - { - \"name\":\"Rocksdb crash test with txn under ASAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug asan_crash_test_with_txn\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $ASAN_CRASH $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make $PARALLELISM asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB test under undefined behavior sanitizer -# -UBSAN_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test under UBSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Test RocksDB debug under UBSAN\", - \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $UBSAN $CLANG $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB crash testing under undefined behavior sanitizer -# -UBSAN_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb crash test under UBSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug ubsan_crash_test\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash testing under undefined behavior sanitizer -# -UBSAN_BLACKBOX_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb blackbox crash test under UBSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug blackbox ubsan_crash_test\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM blackbox_ubsan_crash_test || $CONTRUN_NAME=blackbox_ubsan_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash testing under undefined behavior sanitizer -# -UBSAN_WHITEBOX_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb whitebox crash test under UBSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug whitebox ubsan_crash_test\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM whitebox_ubsan_crash_test || $CONTRUN_NAME=whitebox_ubsan_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash testing with atomic flush under undefined behavior sanitizer -# -UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ - { - \"name\":\"Rocksdb crash test with atomic flush under UBSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug ubsan_crash_test_with_atomic_flush\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash testing with txn under undefined behavior sanitizer -# -UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ - { - \"name\":\"Rocksdb crash test with txn under UBSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and run RocksDB debug ubsan_crash_test_with_txn\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB unit test under valgrind -# -VALGRIND_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test under valgrind\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Run RocksDB debug unit tests\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB test under TSAN -# -TSAN_UNIT_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Unit Test under TSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Run RocksDB debug unit test\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB crash test under TSAN -# -TSAN_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Crash Test under TSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Compile and run\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB blackbox crash test under TSAN -# -TSAN_BLACKBOX_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Blackbox Crash Test under TSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Compile and run\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM blackbox_crash_test || $CONTRUN_NAME=tsan_blackbox_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB whitebox crash test under TSAN -# -TSAN_WHITEBOX_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Whitebox Crash Test under TSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Compile and run\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM whitebox_crash_test || $CONTRUN_NAME=tsan_whitebox_crash_test $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash test with atomic flush under TSAN -# -TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ - { - \"name\":\"Rocksdb Crash Test with atomic flush under TSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Compile and run\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB crash test with txn under TSAN -# -TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ - { - \"name\":\"Rocksdb Crash Test with txn under TSAN\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Compile and run\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make $PARALLELISM crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - $UPLOAD_DB_DIR - ] - $REPORT - } -]" - -# -# RocksDB format compatible -# - -run_format_compatible() -{ - export TEST_TMPDIR=/dev/shm/rocksdb - rm -rf /dev/shm/rocksdb - mkdir /dev/shm/rocksdb - - export https_proxy="fwdproxy:8080" - - tools/check_format_compatible.sh -} - -FORMAT_COMPATIBLE_COMMANDS="[ - { - \"name\":\"Rocksdb Format Compatible tests\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Run RocksDB debug unit test\", - \"shell\":\"cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB no compression -# -run_no_compression() -{ - export TEST_TMPDIR=/dev/shm/rocksdb - rm -rf /dev/shm/rocksdb - mkdir /dev/shm/rocksdb - make clean - cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh - mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh - cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile - mv .tmp.Makefile Makefile - export $SKIP_FORMAT_CHECKS - make $DEBUG $PARALLELISM check -} - -NO_COMPRESSION_COMMANDS="[ - { - \"name\":\"Rocksdb No Compression tests\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Run RocksDB debug unit test\", - \"shell\":\"cd $WORKING_DIR; PARALLEL_J=$PARALLEL_J PARALLEL_j=$PARALLEL_j build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB regression -# -run_regression() -{ - time bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKING_DIR/rocksdb.XXXX) $(mktemp rocksdb_test_stats.XXXX) - - # ======= report size to ODS ======== - - # parameters: $1 -- key, $2 -- value - function send_size_to_ods { - curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ - --connect-timeout 60 - } - - # === normal build === - make clean - make -j$(nproc) static_lib - send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a) - strip librocksdb.a - send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a) - - make clean - make -j$(nproc) shared_lib - send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`) - strip `readlink -f librocksdb.so` - send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) - - # === lite build === - make clean - make LITE=1 -j$(nproc) static_lib - send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a) - strip librocksdb.a - send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a) - - make clean - make LITE=1 -j$(nproc) shared_lib - send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`) - strip `readlink -f librocksdb.so` - send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) -} - -REGRESSION_COMMANDS="[ - { - \"name\":\"Rocksdb regression commands\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Make and run script\", - \"shell\":\"cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB Java build -# -JAVA_BUILD_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Java Build\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build RocksDB for Java\", - \"shell\":\"cd $WORKING_DIR; $SETUP_JAVA_ENV; $SHM make $PARALLELISM rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB fbcode stress/crash test -# -FBCODE_STRESS_CRASH_TEST_COMMANDS="[ - { - \"name\":\"Rocksdb Fbcode Stress and Crash Test\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"timeout\": 86400, - \"steps\": [ - { - \"name\":\"Copy RocksDB code to fbcode repo\", - \"shell\":\"cd internal_repo_rocksdb/repo && git init -b main && git add * && git commit -a -m \\\"Make internal_repo_rocksdb/repo a git repo\\\" && cd ../.. && echo Y | python3 rocks/release_script/release_to_fbcode.py -u internal_repo_rocksdb/repo main || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Build RocksDB fbcode stress tests\", - \"shell\":\"cd $WORKING_DIR; buck build @mode/dbg rocks/tools:rocks_db_stress || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - }, - { - \"name\":\"Run RocksDB whitebox crash tests\", - \"timeout\": 86400, - \"shell\":\"cd $WORKING_DIR; mkdir /dev/shm/rocksdb_fbcode_crash_test && TEST_TMPDIR=\$(mktemp -d --tmpdir=/dev/shm/rocksdb_fbcode_crash_test) python3 rocksdb/src/tools/db_crashtest.py --stress_cmd=buck-out/dbg/gen/rocks/tools/rocks_db_stress -secondary_cache_uri=\\\"$SECONDARY_CACHE_URI\\\" --env_uri=$ENV_URI $EXTRA_DB_STRESS_ARGS -logtostderr=false $TEST_TYPE || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - - -case $1 in - unit) - echo $UNIT_TEST_COMMANDS - ;; - unit_non_shm) - echo $UNIT_TEST_NON_SHM_COMMANDS - ;; - release) - echo $RELEASE_BUILD_COMMANDS - ;; - clang_unit) - echo $CLANG_UNIT_TEST_COMMANDS - ;; - clang_release) - echo $CLANG_RELEASE_BUILD_COMMANDS - ;; - clang_analyze) - echo $CLANG_ANALYZE_COMMANDS - ;; - code_cov) - echo $CODE_COV_COMMANDS - ;; - unity) - echo $UNITY_COMMANDS - ;; - lite) - echo $LITE_BUILD_COMMANDS - ;; - stress_crash) - echo $STRESS_CRASH_TEST_COMMANDS - ;; - blackbox_stress_crash) - echo $BLACKBOX_STRESS_CRASH_TEST_COMMANDS - ;; - whitebox_stress_crash) - echo $WHITEBOX_STRESS_CRASH_TEST_COMMANDS - ;; - stress_crash_with_atomic_flush) - echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS - ;; - stress_crash_with_txn) - echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS - ;; - stress_crash_with_ts) - echo $STRESS_CRASH_TEST_WITH_TS_COMMANDS - ;; - stress_crash_with_multiops_wc_txn) - echo $STRESS_CRASH_TEST_WITH_MULTIOPS_WC_TXN_COMMANDS - ;; - stress_crash_with_multiops_wp_txn) - echo $STRESS_CRASH_TEST_WITH_MULTIOPS_WP_TXN_COMMANDS - ;; - write_stress) - echo $WRITE_STRESS_COMMANDS - ;; - asan) - echo $ASAN_TEST_COMMANDS - ;; - asan_crash) - echo $ASAN_CRASH_TEST_COMMANDS - ;; - blackbox_asan_crash) - echo $ASAN_BLACKBOX_CRASH_TEST_COMMANDS - ;; - whitebox_asan_crash) - echo $ASAN_WHITEBOX_CRASH_TEST_COMMANDS - ;; - asan_crash_with_atomic_flush) - echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS - ;; - asan_crash_with_txn) - echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS - ;; - ubsan) - echo $UBSAN_TEST_COMMANDS - ;; - ubsan_crash) - echo $UBSAN_CRASH_TEST_COMMANDS - ;; - blackbox_ubsan_crash) - echo $UBSAN_BLACKBOX_CRASH_TEST_COMMANDS - ;; - whitebox_ubsan_crash) - echo $UBSAN_WHITEBOX_CRASH_TEST_COMMANDS - ;; - ubsan_crash_with_atomic_flush) - echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS - ;; - ubsan_crash_with_txn) - echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS - ;; - valgrind) - echo $VALGRIND_TEST_COMMANDS - ;; - tsan) - echo $TSAN_UNIT_TEST_COMMANDS - ;; - tsan_crash) - echo $TSAN_CRASH_TEST_COMMANDS - ;; - blackbox_tsan_crash) - echo $TSAN_BLACKBOX_CRASH_TEST_COMMANDS - ;; - whitebox_tsan_crash) - echo $TSAN_WHITEBOX_CRASH_TEST_COMMANDS - ;; - tsan_crash_with_atomic_flush) - echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS - ;; - tsan_crash_with_txn) - echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS - ;; - format_compatible) - echo $FORMAT_COMPATIBLE_COMMANDS - ;; - run_format_compatible) - run_format_compatible - ;; - no_compression) - echo $NO_COMPRESSION_COMMANDS - ;; - run_no_compression) - run_no_compression - ;; - regression) - echo $REGRESSION_COMMANDS - ;; - run_regression) - set -e - run_regression - set +e - ;; - java_build) - echo $JAVA_BUILD_TEST_COMMANDS - ;; - fbcode_stress_crash) - set -f - echo $FBCODE_STRESS_CRASH_TEST_COMMANDS - set +f - ;; - *) - echo "Invalid determinator command" - exit 1 - ;; -esac