From 92731b6b4ab87e64ca05f0e99ca8969ade7a02a5 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Tue, 7 Jul 2020 11:24:00 -0700 Subject: [PATCH] Major CircleCI/Linux fixes / tweaks / enhancements (#7078) Summary: Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get make[1]: write error: stdout probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393 Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem. Significant other changes: * Add another linux build that combines * LIB_MODE=shared, to ensure this works with compile and unit test execution * Alternative rocksdb namespace, to ensure this works (not rely on Travis) * ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it * Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!) * Use gflags with unity and LITE build, as typical case. Debugability improvements: * Use V=1 to show commands being executed (thanks to EAGAIN work-around) * Print kernel version and compiler versions as part of V=1 output from Makefile Cosmetic other changes: * Put more commands on one line, for less clutter in CircleCI output pages * Remove redundant "all" in "make all check" and put make command options before targets * Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078 Reviewed By: siying Differential Revision: D22391647 Pulled By: pdillinger fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135 --- .circleci/cat_ignore_eagain | 54 +++++++++++++++++++++++++++++++++ .circleci/config.yml | 58 ++++++++++++++++++++++------------- Makefile | 60 ++++++++++++++++++------------------- 3 files changed, 121 insertions(+), 51 deletions(-) create mode 100755 .circleci/cat_ignore_eagain diff --git a/.circleci/cat_ignore_eagain b/.circleci/cat_ignore_eagain new file mode 100755 index 000000000..fc0f00d09 --- /dev/null +++ b/.circleci/cat_ignore_eagain @@ -0,0 +1,54 @@ +#! /bin/bash + +# Work around issue with parallel make output causing random error, as in +# make[1]: write error: stdout +# Probably due to a kernel bug: +# https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393 +# Seems to affect image ubuntu-1604:201903-01 and ubuntu-1604:202004-01 + +cd "$(dirname $0)" + +if [ ! -x cat_ignore_eagain.out ]; then + cc -x c -o cat_ignore_eagain.out - << EOF +#include +#include +#include +int main() { + int n, m, p; + char buf[1024]; + for (;;) { + n = read(STDIN_FILENO, buf, 1024); + if (n > 0 && n <= 1024) { + for (m = 0; m < n;) { + p = write(STDOUT_FILENO, buf + m, n - m); + if (p < 0) { + if (errno == EAGAIN) { + // ignore but pause a bit + usleep(100); + } else { + perror("write failed"); + return 42; + } + } else { + m += p; + } + } + } else if (n < 0) { + if (errno == EAGAIN) { + // ignore but pause a bit + usleep(100); + } else { + // Some non-ignorable error + perror("read failed"); + return 43; + } + } else { + // EOF + return 0; + } + } +} +EOF +fi + +exec ./cat_ignore_eagain.out diff --git a/.circleci/config.yml b/.circleci/config.yml index 61b8709d6..ec4483ff5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,10 +18,18 @@ jobs: steps: - checkout # check out the code in the project directory - run: pyenv global 3.5.2 - - run: sudo apt-get update -y - - run: sudo apt-get install -y libgflags-dev - - run: gcc -v - - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 make J=32 all check -j32 + - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain + + build-linux-shared_lib-alt_namespace-status_checked: + machine: + image: ubuntu-1604:201903-01 + resource_class: 2xlarge + steps: + - checkout # check out the code in the project directory + - run: pyenv global 3.5.2 + - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 ASSERT_STATUS_CHECKED=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain build-linux-release: machine: @@ -29,16 +37,21 @@ jobs: resource_class: 2xlarge steps: - checkout # check out the code in the project directory - - run: make release -j32 + - run: make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: if ./db_stress --version; then false; else true; fi # ensure without gflags + - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - run: make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: ./db_stress --version # ensure with gflags build-linux-lite: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:201903-01 resource_class: 2xlarge steps: - checkout # check out the code in the project directory - run: pyenv global 3.5.2 - - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 LITE=1 make J=32 all check -j32 + - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain build-linux-lite-release: machine: @@ -46,7 +59,11 @@ jobs: resource_class: large steps: - checkout # check out the code in the project directory - - run: make release -j32 + - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: if ./db_stress --version; then false; else true; fi # ensure without gflags + - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: ./db_stress --version # ensure with gflags build-linux-clang-no-test: machine: @@ -54,9 +71,8 @@ jobs: resource_class: 2xlarge steps: - checkout # check out the code in the project directory - - run: sudo apt-get update -y - - run: sudo apt-get install -y clang - - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make all -j32 + - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain build-linux-clang10-no-test: machine: @@ -66,9 +82,8 @@ jobs: - checkout # check out the code in the project directory - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: sudo apt-get update -y - - run: sudo apt-get install -y clang-10 - - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make all -j32 # aligned new doesn't work for reason we haven't figured out + - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 all | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out build-linux-clang10-asan: machine: @@ -79,10 +94,8 @@ jobs: - run: pyenv global 3.5.2 - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: sudo apt-get update -y - - run: sudo apt-get install -y clang-10 - - run: sudo apt-get install -y libgflags-dev - - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 make asan_check -j32 # aligned new doesn't work for reason we haven't figured out + - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev + - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out build-linux-cmake: machine: @@ -90,16 +103,16 @@ jobs: resource_class: 2xlarge steps: - checkout # check out the code in the project directory - - run: mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make -j32 + - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain build-linux-unity: docker: # executor type - image: gcc:latest resource_class: xlarge steps: - - run: gcc -v - checkout # check out the code in the project directory - - run: TEST_TMPDIR=/dev/shm && make unity_test -j + - run: apt-get update -y && apt-get install -y libgflags-dev + - run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain build-windows: executor: windows-2xlarge @@ -152,6 +165,9 @@ workflows: build-linux: jobs: - build-linux + build-linux-shared_lib-alt_namespace-status_checked: + jobs: + - build-linux-shared_lib-alt_namespace-status_checked build-linux-lite: jobs: - build-linux-lite diff --git a/Makefile b/Makefile index b93cf1d9b..f3bbfe247 100644 --- a/Makefile +++ b/Makefile @@ -246,11 +246,17 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ export USE_CLANG="$(USE_CLANG)"; \ "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) # this file is generated by the previous line to set build flags and sources - include make_config.mk + export JAVAC_ARGS CLEAN_FILES += make_config.mk +ifeq ($(V), 1) +$(info $(shell uname -a)) +$(info $(shell $(CC) --version)) +$(info $(shell $(CXX) --version)) +endif + missing_make_config_paths := $(shell \ grep "\./\S*\|/\S*" -o $(CURDIR)/make_config.mk | \ while read path; \ @@ -577,10 +583,16 @@ ifdef ASSERT_STATUS_CHECKED work_queue_test \ write_controller_test \ - TESTS := $(filter $(TESTS_PASSING_ASC),$(TESTS)) - PARALLEL_TEST := $(filter $(TESTS_PASSING_ASC),$(PARALLEL_TEST)) + # Enable building all unit tests, but use check_some to run only tests + # known to pass ASC + SUBSET := $(TESTS_PASSING_ASC) + # Alternate: only build unit tests known to pass ASC, and run them + # with make check + #TESTS := $(filter $(TESTS_PASSING_ASC),$(TESTS)) + #PARALLEL_TEST := $(filter $(TESTS_PASSING_ASC),$(PARALLEL_TEST)) +else + SUBSET := $(TESTS) endif -SUBSET := $(TESTS) ifdef ROCKSDBTESTS_START SUBSET := $(shell echo $(SUBSET) | sed 's/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/') endif @@ -706,13 +718,11 @@ benchmarks: $(BENCHMARKS) dbg: $(LIBRARY) $(BENCHMARKS) tools $(TESTS) -# creates static library and programs -release: - $(MAKE) clean - LIB_MODE=$(LIB_MODE) DEBUG_LEVEL=0 $(MAKE) tools db_bench +# creates library and programs +release: clean + LIB_MODE=$(LIB_MODE) DEBUG_LEVEL=0 $(MAKE) $(LIBRARY) tools db_bench -coverage: - $(MAKE) clean +coverage: clean COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check cd coverage && ./coverage_test.sh # Delete intermediate files @@ -951,53 +961,43 @@ whitebox_crash_test_with_txn: db_stress $(PYTHON) -u tools/db_crashtest.py --txn whitebox --random_kill_odd \ $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) -asan_check: - $(MAKE) clean +asan_check: clean COMPILE_WITH_ASAN=1 $(MAKE) check -j32 $(MAKE) clean -asan_crash_test: - $(MAKE) clean +asan_crash_test: clean COMPILE_WITH_ASAN=1 $(MAKE) crash_test $(MAKE) clean -asan_crash_test_with_atomic_flush: - $(MAKE) clean +asan_crash_test_with_atomic_flush: clean COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_atomic_flush $(MAKE) clean -asan_crash_test_with_txn: - $(MAKE) clean +asan_crash_test_with_txn: clean COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_txn $(MAKE) clean -asan_crash_test_with_best_efforts_recovery: - $(MAKE) clean +asan_crash_test_with_best_efforts_recovery: clean COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_best_efforts_recovery $(MAKE) clean -ubsan_check: - $(MAKE) clean +ubsan_check: clean COMPILE_WITH_UBSAN=1 $(MAKE) check -j32 $(MAKE) clean -ubsan_crash_test: - $(MAKE) clean +ubsan_crash_test: clean COMPILE_WITH_UBSAN=1 $(MAKE) crash_test $(MAKE) clean -ubsan_crash_test_with_atomic_flush: - $(MAKE) clean +ubsan_crash_test_with_atomic_flush: clean COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_atomic_flush $(MAKE) clean -ubsan_crash_test_with_txn: - $(MAKE) clean +ubsan_crash_test_with_txn: clean COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_txn $(MAKE) clean -ubsan_crash_test_with_best_efforts_recovery: - $(MAKE) clean +ubsan_crash_test_with_best_efforts_recovery: clean COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_best_efforts_recovery $(MAKE) clean