CircleCI build improvements (#7392)

Summary:
* add slack integration
* parallelize `check_some` tests with gtest-parallel
* upgrade build image to the latest one: ubuntu-1604:202007-01
* clean up the config

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7392

Reviewed By: pdillinger

Differential Revision: D23845379

Pulled By: jay-zhuang

fbshipit-source-id: f3af82dc2daeba6441f61c858e08d3936c6ccd10

.circleci/config.yml

@@ -2,6 +2,59 @@ version: 2.1
 
 orbs:
   win: circleci/windows@2.4.0
+  slack: circleci/slack@3.4.2
+
+aliases:
+  - &notify-on-master-failure
+    fail_only: true
+    only_for_branches: master
+
+commands:
+  pre-steps:
+    steps:
+      - checkout
+      - run: pyenv global 3.5.9
+      - run:
+          name: Setup Environment Variables
+          command: |
+            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
+            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
+            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
+            echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
+
+  post-steps:
+    steps:
+      - slack/status: *notify-on-master-failure
+      - store_test_results: # store test result if there's any
+          path: /tmp/test-results
+      - store_artifacts: # store LOG for debugging if there's any
+          path: LOG
+
+  install-clang-10:
+    steps:
+      - run:
+          name: Install Clang 10
+          command: |
+            echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+            echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
+            sudo apt-get update -y && sudo apt-get install -y clang-10
+
+  install-gflags:
+    steps:
+      - run:
+          name: Install gflags
+          command: |
+            sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+
+  install-gtest-parallel:
+    steps:
+      - run:
+          name: Install gtest-parallel
+          command: |
+            echo "Currently using a forked version to print task status in circleCI output, pending: https://github.com/google/gtest-parallel/pull/79"
+            git clone --single-branch --branch master --depth 1 https://github.com/jay-zhuang/gtest-parallel.git ~/gtest-parallel
+            echo "export PATH=$HOME/gtest-parallel:$PATH" >> $BASH_ENV
 
 executors:
   windows-2xlarge:
@@ -13,39 +66,36 @@ executors:
 jobs:
   build-linux:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+      - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
+      - post-steps
-      - store_test_results:
-          path: /tmp/test-results
 
   build-linux-shared_lib-alt_namespace-status_checked:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain
+      - post-steps
-      - store_test_results:
-          path: /tmp/test-results
 
   build-linux-release:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
       - checkout # check out the code in the project directory
       - run: make V=1 -j32 release | .circleci/cat_ignore_eagain
       - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+      - install-gflags
       - run: make V=1 -j32 release | .circleci/cat_ignore_eagain
       - run: ./db_stress --version # ensure with gflags
+      - post-steps
 
   build-linux-release-rtti:
     machine:
@@ -63,112 +113,90 @@ jobs:
 
   build-linux-lite:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+      - run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 LITE=1 GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
+      - post-steps
-      - store_test_results:
-          path: /tmp/test-results
 
   build-linux-lite-release:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: large
     steps:
       - checkout # check out the code in the project directory
       - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
       - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+      - install-gflags
       - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
       - run: ./db_stress --version # ensure with gflags
+      - post-steps
 
   build-linux-clang-no-test:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
       - checkout # check out the code in the project directory
       - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
       - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain
-
+      - post-steps
-  build-linux-clang10-no-test:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: 2xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-      - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-      - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev
-      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 all | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
 
   build-linux-clang10-asan:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - install-clang-10
-      - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
+      - post-steps
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - store_test_results:
-          path: /tmp/test-results
 
   build-linux-clang10-mini-tsan:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - install-clang-10
-      - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 EXCLUDE_TESTS_REGEX="TransactionStressTest|SnapshotConcurrentAccess|SeqAdvanceConcurrent|DeadlockStress|MultiThreadedDBTest.MultiThreaded|WriteUnpreparedStressTest.ReadYourOwnWriteStress|DBAsBaseDB/TransactionStressTest|FlushCloseWALFiles|BackgroundPurgeCFDropTest" make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. Exclude FlushCloseWALFiles and BackgroundPurgeCFDropTest for occasional failures.
-      - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
+      - post-steps
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 EXCLUDE_TESTS_REGEX="TransactionStressTest|SnapshotConcurrentAccess|SeqAdvanceConcurrent|DeadlockStress|MultiThreadedDBTest.MultiThreaded|WriteUnpreparedStressTest.ReadYourOwnWriteStress|DBAsBaseDB/TransactionStressTest|FlushCloseWALFiles|BackgroundPurgeCFDropTest" GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. Exclude FlushCloseWALFiles and BackgroundPurgeCFDropTest for occasional failures.
-      - store_test_results:
-          path: /tmp/test-results
 
   build-linux-clang10-ubsan:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - install-clang-10
-      - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
+      - post-steps
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
 
   build-linux-clang10-clang-analyze:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - install-clang-10
-      - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
+      - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
-      - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
+      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev clang-tools-10
+      - post-steps
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
 
   build-linux-cmake:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
       - checkout # check out the code in the project directory
       - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
+      - post-steps
 
   build-linux-unity:
     docker: # executor type
@@ -178,16 +206,17 @@ jobs:
       - checkout # check out the code in the project directory
       - run: apt-get update -y && apt-get install -y libgflags-dev
       - run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain
+      - post-steps
 
   build-linux-gcc-4-8:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: large
     steps:
-      - checkout
+      - pre-steps
-      - run: pyenv global 3.5.2
       - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
       - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - post-steps
 
   build-windows:
     executor: windows-2xlarge
@@ -252,38 +281,38 @@ jobs:
 
   build-linux-java:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     steps:
-      - checkout
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
       - run:
           name: "Build RocksDBJava"
           command: |
-            export JAVA_HOME=/usr/lib/jvm/jdk1.8.0
+            export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
             export PATH=$JAVA_HOME/bin:$PATH
             echo "JAVA_HOME=${JAVA_HOME}"
             which java && java -version
             which javac && javac -version
-            SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 J=32 -j32 rocksdbjava jtest | .circleci/cat_ignore_eagain
+            make V=1 J=32 -j32 rocksdbjava jtest | .circleci/cat_ignore_eagain
+      - post-steps
 
   build-examples:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: medium
     steps:
-      - checkout # check out the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
       - run:
           name: "Build examples"
           command: |
             OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain
+      - post-steps
 
   build-linux-non-shm:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     resource_class: 2xlarge
     parameters:
       start_test:
@@ -293,13 +322,23 @@ jobs:
         default: ""
         type: string
     steps:
-      - checkout # checkout the code in the project directory
+      - pre-steps
-      - run: pyenv global 3.5.2
+      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+      - install-gtest-parallel
       - run:
-          name: "Build and unit tests on non-shm"
+          name: "Build unit tests"
           command: |
-            TMPD=/tmp/rocksdb_test_tmp ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> make V=1 J=32 -j32 check_some | .circleci/cat_ignore_eagain
+            echo "env: $(env)"
+            echo "** done env"
+            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
+      - run:
+          name: "Run unit tests in parallel"
+          command: |
+            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
+            cat /tmp/test_list
+            export TEST_TMPDIR=/tmp/rocksdb_test_tmp
+            /usr/bin/python ../gtest-parallel/gtest-parallel $(</tmp/test_list) --output_dir=/tmp --print_task_status_continuously
+      - post-steps
 
 workflows:
   build-linux:
@@ -323,9 +362,6 @@ workflows:
   build-linux-clang-no-test:
     jobs:
       - build-linux-clang-no-test
-  build-linux-clang10-no-test:
-    jobs:
-      - build-linux-clang10-no-test
   build-linux-clang10-asan:
     jobs:
       - build-linux-clang10-asan
@@ -367,25 +403,19 @@ workflows:
   build-examples:
     jobs:
       - build-examples
-  build-linux-non-shm-1:
+  build-linux-non-shm:
     jobs:
       - build-linux-non-shm:
           start_test: ""
-          end_test: "db_tailing_iter_test" # make sure unique in src.mk
+          end_test: "db_options_test" # make sure unique in src.mk
-  build-linux-non-shm-2:
-    jobs:
       - build-linux-non-shm:
-          start_test: "db_tailing_iter_test" # make sure unique in src.mk
+          start_test: "db_options_test" # make sure unique in src.mk
-          end_test: "db_test2" # make sure unique in src.mk
+          end_test: "filename_test" # make sure unique in src.mk
-  build-linux-non-shm-3:
-    jobs:
       - build-linux-non-shm:
-          start_test: "db_test2" # make sure unique in src.mk
+          start_test: "filename_test" # make sure unique in src.mk
-          end_test: "arena_test" # make sure unique in src.mk
+          end_test: "statistics_test" # make sure unique in src.mk
-  build-linux-non-shm-4:
-    jobs:
       - build-linux-non-shm:
-          start_test: "compact_on_deletion_collector_test" # make sure unique in src.mk
+          start_test: "statistics_test" # make sure unique in src.mk
           end_test: ""
   build-linux-gcc-4-8:
     jobs:

Makefile

@@ -2329,6 +2329,9 @@ else
 depend: $(DEPFILES)
 endif
 
+build_subset_tests: $(ROCKSDBTESTS_SUBSET)
+	$(AM_V_GEN)if [ -n "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}" ]; then echo "$(ROCKSDBTESTS_SUBSET)" > "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}"; else echo "$(ROCKSDBTESTS_SUBSET)"; fi
+
 # if the make goal is either "clean" or "format", we shouldn't
 # try to import the *.d files.
 # TODO(kailiu) The unfamiliarity of Make's conditions leads to the ugly