From eda83eaac0ed7ee9078c7cd4fb0409c5f2e28ebe Mon Sep 17 00:00:00 2001 From: Jay Zhuang Date: Tue, 1 Jun 2021 14:42:06 -0700 Subject: [PATCH] Fix cmake build failure with gflags (#8324) Summary: - Fix cmake build failure with gflags. - Add CI tests for both gflags 2.1 and 2.2. - Fix ctest config with gtest. - Add CI to run test with ctest. One benefit of ctest is it support timeout, it's set to 5min in our CI, so we will know which test is hang. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8324 Test Plan: CI pass Reviewed By: ajkr Differential Revision: D28762517 Pulled By: jay-zhuang fbshipit-source-id: 09063c5af5f9f33abfcdeb48593acbd9826cd199 --- .circleci/config.yml | 81 ++++++++++++------- CMakeLists.txt | 27 ++++--- .../block_cache_trace_analyzer_test.cc | 2 +- tools/io_tracer_parser_test.cc | 2 +- tools/trace_analyzer_test.cc | 2 +- 5 files changed, 74 insertions(+), 40 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc904fdca..8c8abfcd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,6 +49,9 @@ commands: echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV + echo "export GTEST_COLOR=1" >> $BASH_ENV + echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV + echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV pre-steps-macos: steps: @@ -88,6 +91,14 @@ commands: command: | sudo apt-get update -y && sudo apt-get install -y libgflags-dev + upgrade-cmake: + steps: + - run: + name: Upgrade cmake + command: | + sudo apt remove --purge cmake + sudo snap install cmake --classic + install-gflags-on-macos: steps: - run: @@ -140,12 +151,12 @@ jobs: - install-cmake-on-macos - install-gflags-on-macos - pre-steps-macos - - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32) | .circleci/cat_ignore_eagain + - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) | .circleci/cat_ignore_eagain - post-steps build-linux: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -155,7 +166,7 @@ jobs: build-linux-mem-env: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -165,7 +176,7 @@ jobs: build-linux-encrypted-env: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -175,7 +186,7 @@ jobs: build-linux-shared_lib-alt_namespace-status_checked: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -185,7 +196,7 @@ jobs: build-linux-release: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: large steps: - checkout # check out the code in the project directory @@ -212,7 +223,7 @@ jobs: build-linux-lite: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -222,7 +233,7 @@ jobs: build-linux-lite-release: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: large steps: - checkout # check out the code in the project directory @@ -235,7 +246,7 @@ jobs: build-linux-clang-no_test_run: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: xlarge steps: - checkout # check out the code in the project directory @@ -245,7 +256,7 @@ jobs: build-linux-clang10-asan: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -256,7 +267,7 @@ jobs: build-linux-clang10-mini-tsan: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -267,7 +278,7 @@ jobs: build-linux-clang10-ubsan: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -278,7 +289,7 @@ jobs: build-linux-clang10-clang-analyze: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -288,13 +299,25 @@ jobs: - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. - post-steps - build-linux-cmake-no_test_run: + build-linux-cmake: machine: - image: ubuntu-1604:202007-01 - resource_class: large + image: ubuntu-1604:202104-01 + resource_class: 2xlarge + steps: + - checkout # check out the code in the project directory + - install-gflags + - upgrade-cmake + - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain + - post-steps + + build-linux-cmake-ubuntu-20: + machine: + image: ubuntu-2004:202104-01 + resource_class: 2xlarge steps: - checkout # check out the code in the project directory - - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain + - install-gflags + - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain - post-steps build-linux-unity: @@ -309,7 +332,7 @@ jobs: build-linux-gcc-4_8-no_test_run: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: large steps: - pre-steps @@ -417,7 +440,7 @@ jobs: build-linux-java: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: large environment: JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64 @@ -441,7 +464,7 @@ jobs: build-linux-java-static: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: large environment: JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64 @@ -512,7 +535,7 @@ jobs: build-examples: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: large steps: - pre-steps @@ -525,7 +548,7 @@ jobs: build-cmake-mingw: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -548,7 +571,7 @@ jobs: build-linux-non-shm: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge parameters: start_test: @@ -604,6 +627,7 @@ jobs: JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64 steps: - pre-steps + - install-gflags - run: name: "Set Java Environment" command: | @@ -616,7 +640,7 @@ jobs: command: | mkdir build cd build - cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. + cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. make -j4 - run: name: "Build Java with cmake" @@ -624,13 +648,13 @@ jobs: rm -rf build mkdir build cd build - cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=0 .. + cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 .. make -j4 rocksdb rocksdbjni - post-steps build-format-compatible: machine: - image: ubuntu-1604:202007-01 + image: ubuntu-1604:202104-01 resource_class: 2xlarge steps: - pre-steps @@ -650,6 +674,10 @@ workflows: build-linux: jobs: - build-linux + build-linux-cmake: + jobs: + - build-linux-cmake + - build-linux-cmake-ubuntu-20 build-linux-mem-env: jobs: - build-linux-mem-env @@ -733,7 +761,6 @@ workflows: build-linux-compilers-no_test_run: jobs: - build-linux-clang-no_test_run - - build-linux-cmake-no_test_run - build-linux-gcc-4_8-no_test_run - build-linux-gcc-8-no_test_run - build-linux-gcc-9-no_test_run diff --git a/CMakeLists.txt b/CMakeLists.txt index 6172010c5..61d94a709 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,10 +32,11 @@ # 3. cmake .. # 4. make -j -cmake_minimum_required(VERSION 3.5.1) +cmake_minimum_required(VERSION 3.10) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/") include(ReadVersion) +include(GoogleTest) get_rocksdb_version(rocksdb_VERSION) project(rocksdb VERSION ${rocksdb_VERSION} @@ -71,6 +72,12 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, if (WITH_WINDOWS_UTF8_FILENAMES) add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES) endif() + +if ($ENV{CIRCLECI}) + message(STATUS "Build for CircieCI env, a few tests may be disabled") + add_definitions(-DCIRCLECI) +endif() + # third-party/folly is only validated to work on Linux and Windows for now. # So only turn it on there by default. if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows") @@ -123,11 +130,11 @@ else() set(GFLAGS_LIB ${GFLAGS_TARGET}) else() # Config with GFLAGS_LIBRARIES available since gflags 2.1.0 - set(GFLAGS_LIB ${GFLAGS_LIBRARIES}) + set(GFLAGS_LIB ${gflags_LIBRARIES}) endif() else() find_package(gflags REQUIRED) - set(GFLAGS_LIB gflags::gflags) + set(GFLAGS_LIB gflags::gflags) endif() include_directories(${GFLAGS_INCLUDE_DIR}) list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB}) @@ -211,7 +218,7 @@ if(MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324") else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing") if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes") @@ -1321,21 +1328,21 @@ if(WITH_TESTS) foreach(sourcefile ${TESTS}) get_filename_component(exename ${sourcefile} NAME_WE) - add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile}) - set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} + add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}) + set_target_properties(${exename}${ARTIFACT_SUFFIX} PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1 EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1 EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1 OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX} ) - target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB}) + target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB}) if(NOT "${exename}" MATCHES "db_sanity_test") - add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX}) - add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}) + gtest_discover_tests(${exename}) + add_dependencies(check ${exename}${ARTIFACT_SUFFIX}) endif() if("${exename}" MATCHES "env_librados_test") # env_librados_test.cc uses librados directly - target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} rados) + target_link_libraries(${exename}${ARTIFACT_SUFFIX} rados) endif() endforeach(sourcefile ${TESTS}) diff --git a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc index 84e2fe521..91bd30652 100644 --- a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +++ b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc @@ -9,7 +9,7 @@ int main() { fprintf(stderr, "Please install gflags to run block_cache_trace_analyzer_test\n"); - return 1; + return 0; } #else diff --git a/tools/io_tracer_parser_test.cc b/tools/io_tracer_parser_test.cc index 845d7569b..d91217341 100644 --- a/tools/io_tracer_parser_test.cc +++ b/tools/io_tracer_parser_test.cc @@ -9,7 +9,7 @@ #include int main() { fprintf(stderr, "Please install gflags to run io_tracer_parser_test\n"); - return 1; + return 0; } #else diff --git a/tools/trace_analyzer_test.cc b/tools/trace_analyzer_test.cc index 6a035b24a..e016ea591 100644 --- a/tools/trace_analyzer_test.cc +++ b/tools/trace_analyzer_test.cc @@ -12,7 +12,7 @@ #include int main() { fprintf(stderr, "Please install gflags to run trace_analyzer test\n"); - return 1; + return 0; } #else