From 67542bfab5d052807e13aaaf8bcc0fa849375722 Mon Sep 17 00:00:00 2001 From: Jay Zhuang Date: Thu, 3 Mar 2022 11:58:51 -0800 Subject: [PATCH] Improve build speed (#9605) Summary: Improve the CI build speed: - split the macos tests to 2 parallel jobs - split tsan tests to 2 parallel jobs - move non-shm tests to nightly build - slow jobs use lager machine - fast jobs use smaller machine - add microbench to no-test jobs - add run-microbench to nightly build Pull Request resolved: https://github.com/facebook/rocksdb/pull/9605 Reviewed By: riversand963 Differential Revision: D34358982 Pulled By: jay-zhuang fbshipit-source-id: d5091b3f4ef6d25c5c37920fb614f3342ee60e4a --- .circleci/config.yml | 156 ++++++++++++++++++++++++------------------- Makefile | 5 ++ 2 files changed, 91 insertions(+), 70 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6c694aba..a7ae34e8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,7 +87,6 @@ commands: - run: name: Install Clang 13 command: | - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable @@ -171,19 +170,42 @@ jobs: - increase-max-open-files-on-macos - install-gflags-on-macos - pre-steps-macos - - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check + - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 all - post-steps build-macos-cmake: macos: xcode: 12.5.1 resource_class: large + parameters: + run_even_tests: + description: run even or odd tests, used to split tests to 2 groups + type: boolean + default: true steps: - increase-max-open-files-on-macos - install-cmake-on-macos - install-gflags-on-macos - pre-steps-macos - - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) + - run: + name: "cmake generate project file" + command: ulimit -S -n 1048576 && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. + - run: + name: "Build tests" + command: cd build && make V=1 -j32 + - when: + condition: << parameters.run_even_tests >> + steps: + - run: + name: "Run even tests" + command: ulimit -S -n 1048576 && cd build && ctest -j32 -I 0,,2 + - when: + condition: + not: << parameters.run_even_tests >> + steps: + - run: + name: "Run odd tests" + command: ulimit -S -n 1048576 && cd build && ctest -j32 -I 1,,2 - post-steps build-linux: @@ -219,38 +241,38 @@ jobs: build-linux-release: machine: image: ubuntu-2004:202111-02 - resource_class: large + resource_class: 2xlarge steps: - checkout # check out the code in the project directory - - run: make V=1 -j8 release + - run: make V=1 -j32 release - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - install-gflags - - run: make V=1 -j8 release + - run: make V=1 -j32 release - run: ./db_stress --version # ensure with gflags - post-steps build-linux-release-rtti: machine: image: ubuntu-2004:202111-02 - resource_class: large + resource_class: xlarge steps: - checkout # check out the code in the project directory - run: make clean - - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev - run: make clean - - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench - run: ./db_stress --version # ensure with gflags build-linux-lite: machine: image: ubuntu-2004:202111-02 - resource_class: 2xlarge + resource_class: large steps: - pre-steps - install-gflags - - run: LITE=1 make V=1 J=32 -j32 check + - run: LITE=1 make V=1 J=8 -j8 check - post-steps build-linux-lite-release: @@ -291,11 +313,30 @@ jobs: machine: image: ubuntu-2004:202111-02 resource_class: 2xlarge + # find test list by `make list_all_tests` + parameters: + start_test: + default: "" + type: string + end_test: + default: "" + type: string steps: - pre-steps - install-gflags - install-clang-10 - - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out. + - install-gtest-parallel + - run: + name: "Build unit tests" + command: | + echo "env: $(env)" + ROCKSDBTESTS_START=<> ROCKSDBTESTS_END=<> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 --output-sync=target build_subset_tests + - run: + name: "Run unit tests in parallel" + command: | + sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list + cat /tmp/test_list + gtest-parallel $(> .. cd .. echo "Building with VS version: ${CMAKE_GENERATOR}" - msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + msbuild.exe build/rocksdb.sln -maxCpuCount:64 -property:Configuration=Debug -property:Platform=x64 - run: name: "Test RocksDB" shell: powershell.exe @@ -531,7 +574,7 @@ jobs: build-macos-java: macos: xcode: 12.5.1 - resource_class: medium + resource_class: large environment: JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash @@ -549,16 +592,16 @@ jobs: which javac && javac -version - run: name: "Build RocksDBJava Shared Library" - command: make V=1 J=8 -j8 rocksdbjava + command: make V=1 J=16 -j16 rocksdbjava - run: name: "Test RocksDBJava" - command: make V=1 J=8 -j8 jtest + command: make V=1 J=16 -j16 jtest - post-steps build-macos-java-static: macos: xcode: 12.5.1 - resource_class: medium + resource_class: large environment: JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home steps: @@ -576,13 +619,13 @@ jobs: which javac && javac -version - run: name: "Build RocksDBJava x86 and ARM Static Libraries" - command: make V=1 J=8 -j8 rocksdbjavastaticosx + command: make V=1 J=16 -j16 rocksdbjavastaticosx - post-steps build-macos-java-static-universal: macos: xcode: 12.5.1 - resource_class: medium + resource_class: large environment: JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home steps: @@ -600,7 +643,7 @@ jobs: which javac && javac -version - run: name: "Build RocksDBJava Universal Binary Static Library" - command: make V=1 J=8 -j8 rocksdbjavastaticosx_ub + command: make V=1 J=16 -j16 rocksdbjavastaticosx_ub - post-steps build-examples: @@ -619,7 +662,7 @@ jobs: build-cmake-mingw: machine: image: ubuntu-2004:202111-02 - resource_class: 2xlarge + resource_class: large steps: - pre-steps - install-gflags @@ -643,30 +686,12 @@ jobs: machine: image: ubuntu-2004:202111-02 resource_class: 2xlarge - parameters: - start_test: - default: "" - type: string - end_test: - default: "" - type: string + environment: + TEST_TMPDIR: /tmp/rocksdb_test_tmp steps: - pre-steps - install-gflags - - install-gtest-parallel - - run: - name: "Build unit tests" - command: | - echo "env: $(env)" - echo "** done env" - ROCKSDBTESTS_START=<> ROCKSDBTESTS_END=<> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests - - run: - name: "Run unit tests in parallel" - command: | - sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list - cat /tmp/test_list - export TEST_TMPDIR=/tmp/rocksdb_test_tmp - gtest-parallel $( "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}"; else echo "$(ROCKSDBTESTS_SUBSET)"; fi +list_all_tests: + echo "$(ROCKSDBTESTS_SUBSET)" + # Remove the rules for which dependencies should not be generated and see if any are left. #If so, include the dependencies; if not, do not include the dependency files ROCKS_DEP_RULES=$(filter-out clean format check-format check-buck-targets check-headers check-sources jclean jtest package analyze tags rocksdbjavastatic% unity.% unity_test, $(MAKECMDGOALS))