NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Testing out parallel sandcastle changes

Browse Source 
Summary:
Removing moreutils from sandcastle and adding gnu parallel.
Then passing in J= nproc command

Test Plan: Testing on sandcastle

Reviewers: sdong, kradhakrishnan

Reviewed By: kradhakrishnan

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D61017
main
Anirban Rahut 9 years ago
parent 7efd9c25c4
commit d3bfd33972
5 changed files with 7978 additions and 25 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 16
      Makefile
  2. 5
      arcanist_util/config/RocksDBCommonHelper.php
  3. 7936
      build_tools/gnu_parallel
  4. 2
      build_tools/precommit_checker.py
  5. 44
      build_tools/rocksdb-lego-determinator

16
Makefile
Unescape View File

@ -621,7 +621,7 @@ check_0:
} \ } \
| $(prioritize_long_running_tests) \ | $(prioritize_long_running_tests) \
| grep -E '$(tests-regexp)' \ | grep -E '$(tests-regexp)' \
| parallel -j$(J) --joblog=LOG $$eta --gnu '{} >& t/log-{/}' | build_tools/gnu_parallel -j$(J) --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
.PHONY: valgrind_check_0 .PHONY: valgrind_check_0
valgrind_check_0: valgrind_check_0:
@ -636,7 +636,7 @@ valgrind_check_0:
} \ } \
| $(prioritize_long_running_tests) \ | $(prioritize_long_running_tests) \
| grep -E '$(tests-regexp)' \ | grep -E '$(tests-regexp)' \
| parallel -j$(J) --joblog=LOG $$eta --gnu \ | build_tools/gnu_parallel -j$(J) --joblog=LOG $$eta --gnu \
'if [[ "{}" == "./"* ]] ; then $(DRIVER) {} >& t/valgrind_log-{/}; ' \ 'if [[ "{}" == "./"* ]] ; then $(DRIVER) {} >& t/valgrind_log-{/}; ' \
'else {} >& t/valgrind_log-{/}; fi' 'else {} >& t/valgrind_log-{/}; fi'
@ -657,7 +657,7 @@ watch-log:
check: all check: all
$(MAKE) gen_parallel_tests $(MAKE) gen_parallel_tests
$(AM_V_GEN)if test "$(J)" != 1 \ $(AM_V_GEN)if test "$(J)" != 1 \
&& (parallel --gnu --help 2>/dev/null) | \ && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
grep -q 'GNU Parallel'; \ grep -q 'GNU Parallel'; \
then \ then \
$(MAKE) T="$$t" TMPD=$(TMPD) check_0; \ $(MAKE) T="$$t" TMPD=$(TMPD) check_0; \
@ -717,7 +717,7 @@ ubsan_crash_test:
valgrind_check: $(TESTS) valgrind_check: $(TESTS)
$(MAKE) gen_parallel_tests $(MAKE) gen_parallel_tests
$(AM_V_GEN)if test "$(J)" != 1 \ $(AM_V_GEN)if test "$(J)" != 1 \
&& (parallel --gnu --help 2>/dev/null) | \ && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
grep -q 'GNU Parallel'; \ grep -q 'GNU Parallel'; \
then \ then \
$(MAKE) TMPD=$(TMPD) \ $(MAKE) TMPD=$(TMPD) \
@ -739,10 +739,10 @@ parloop:
for t in $(PAR_TEST); do \ for t in $(PAR_TEST); do \
echo "===== Running $$t in parallel $(NUM_PAR)";\ echo "===== Running $$t in parallel $(NUM_PAR)";\
if [ $(db_test) -eq 1 ]; then \ if [ $(db_test) -eq 1 ]; then \
seq $(J) | v="$$t" parallel --gnu 's=$(TMPD)/rdb-{}; export TEST_TMPDIR=$$s;' \ seq $(J) | v="$$t" build_tools/gnu_parallel --gnu 's=$(TMPD)/rdb-{}; export TEST_TMPDIR=$$s;' \
'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \ 'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \
else\ else\
seq $(J) | v="./$$t" parallel --gnu 's=$(TMPD)/rdb-{};' \ seq $(J) | v="./$$t" build_tools/gnu_parallel --gnu 's=$(TMPD)/rdb-{};' \
'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \ 'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \
fi; \ fi; \
ret_code=$$?; \ ret_code=$$?; \
@ -763,7 +763,7 @@ test_names = \
parallel_check: $(TESTS) parallel_check: $(TESTS)
$(AM_V_GEN)if test "$(J)" > 1 \ $(AM_V_GEN)if test "$(J)" > 1 \
&& (parallel --gnu --help 2>/dev/null) | \ && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
grep -q 'GNU Parallel'; \ grep -q 'GNU Parallel'; \
then \ then \
echo Running in parallel $(J); \ echo Running in parallel $(J); \
@ -773,7 +773,7 @@ parallel_check: $(TESTS)
ret_bad=0; \ ret_bad=0; \
echo $(J);\ echo $(J);\
echo Test Dir: $(TMPD); \ echo Test Dir: $(TMPD); \
seq $(J) | parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \ seq $(J) | build_tools/gnu_parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
$(MAKE) PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \ $(MAKE) PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \
J=$(J) db_test=1 parloop; \ J=$(J) db_test=1 parloop; \
$(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \ $(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \

5
arcanist_util/config/RocksDBCommonHelper.php
Unescape View File

@ -75,9 +75,10 @@ function getSteps($applyDiff, $diffID, $username, $test) {
); );
// arc demands certain permission on its config. // arc demands certain permission on its config.
// also fix the sticky bit issue in sandcastle
$fix_permission = array( $fix_permission = array(
"name" => "Fix environment", "name" => "Fix environment",
"shell" => "chmod 600 ~/.arcrc", "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
"user" => "root" "user" => "root"
); );
@ -115,7 +116,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
} }
// Run the actual command. // Run the actual command.
$cmd = $cmd . "./build_tools/precommit_checker.py " . $test $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
. "; exit_code=$?; "; . "; exit_code=$?; ";
if ($applyDiff) { if ($applyDiff) {

7936
build_tools/gnu_parallel
View File

File diff suppressed because it is too large Load Diff

2
build_tools/precommit_checker.py
Unescape View File

@ -94,7 +94,7 @@ class PreCommitChecker(Env):
# #
def get_commands(self, test): def get_commands(self, test):
status, out = self.GetOutput( status, out = self.GetOutput(
"build_tools/rocksdb-lego-determinator %s" % test, ".") "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
return status, out return status, out
# #

44
build_tools/rocksdb-lego-determinator
Unescape View File

@ -56,10 +56,26 @@ fi
CLEANUP_ENV=" CLEANUP_ENV="
{ {
'name':'Cleanup environment', 'name':'Cleanup environment',
'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean; chmod -t /dev/shm', 'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && chmod +t /dev/shm && make clean',
'user':'root' 'user':'root'
}" }"
# We will eventually set the RATIO to 1, but we want do this
# in steps. RATIO=$(nproc) will make it work as J=1
if [ -z $RATIO ]; then
RATIO=$(nproc)
fi
if [ -z $PARALLEL_J ]; then
PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
fi
if [ -z $PARALLEL_j ]; then
PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
fi
PARALLELISM="$PARALLEL_J $PARALLEL_j"
DEBUG="OPT=-g" DEBUG="OPT=-g"
SHM="TEST_TMPDIR=/dev/shm/rocksdb" SHM="TEST_TMPDIR=/dev/shm/rocksdb"
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
@ -137,7 +153,7 @@ UNIT_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build and test RocksDB debug version', 'name':'Build and test RocksDB debug version',
'shell':'$SHM $DEBUG make J=1 check', 'shell':'$SHM $DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -159,7 +175,7 @@ UNIT_TEST_NON_SHM_COMMANDS="[
{ {
'name':'Build and test RocksDB debug version', 'name':'Build and test RocksDB debug version',
'timeout': 86400, 'timeout': 86400,
'shell':'$DEBUG make J=1 check', 'shell':'$DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -179,7 +195,7 @@ RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build RocksDB release', 'name':'Build RocksDB release',
'shell':'make J=1 release', 'shell':'make $PARALLEL_j release',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -199,7 +215,7 @@ UNIT_TEST_COMMANDS_481="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build and test RocksDB debug version', 'name':'Build and test RocksDB debug version',
'shell':'$SHM $GCC_481 $DEBUG make J=1 check', 'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -219,7 +235,7 @@ RELEASE_BUILD_COMMANDS_481="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build RocksDB release on GCC 4.8.1', 'name':'Build RocksDB release on GCC 4.8.1',
'shell':'$GCC_481 make J=1 release', 'shell':'$GCC_481 make $PARALLEL_j release',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -239,7 +255,7 @@ CLANG_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build and test RocksDB debug', 'name':'Build and test RocksDB debug',
'shell':'$CLANG $SHM $DEBUG make J=1 check', 'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -259,7 +275,7 @@ CLANG_RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build RocksDB release', 'name':'Build RocksDB release',
'shell':'$CLANG make J=1 release', 'shell':'$CLANG make $PARALLEL_j release',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -279,7 +295,7 @@ CLANG_ANALYZE_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'RocksDB build and analyze', 'name':'RocksDB build and analyze',
'shell':'$CLANG $SHM $DEBUG make J=1 analyze', 'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -299,7 +315,7 @@ CODE_COV_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build, test and collect code coverage info', 'name':'Build, test and collect code coverage info',
'shell':'$SHM $DEBUG make J=1 coverage', 'shell':'$SHM $DEBUG make $PARALLELISM coverage',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -430,7 +446,7 @@ ASAN_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Test RocksDB debug under ASAN', 'name':'Test RocksDB debug under ASAN',
'shell':'set -o pipefail && $SHM $ASAN $DEBUG make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d', 'shell':'set -o pipefail && $SHM $ASAN $DEBUG make $PARALLELISM asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'user':'root', 'user':'root',
$PARSER $PARSER
} }
@ -472,7 +488,7 @@ UBSAN_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Test RocksDB debug under UBSAN', 'name':'Test RocksDB debug under UBSAN',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check', 'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check',
'user':'root', 'user':'root',
$PARSER $PARSER
} }
@ -516,7 +532,7 @@ VALGRIND_TEST_COMMANDS="[
{ {
'name':'Run RocksDB debug unit tests', 'name':'Run RocksDB debug unit tests',
'timeout': 86400, 'timeout': 86400,
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make valgrind_check', 'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make $PARALLELISM valgrind_check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -538,7 +554,7 @@ TSAN_UNIT_TEST_COMMANDS="[
{ {
'name':'Run RocksDB debug unit test', 'name':'Run RocksDB debug unit test',
'timeout': 86400, 'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 check', 'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },

Write Preview
Loading…
Cancel
Save