Fix cmake build failure with gflags (#8324)

Summary: - Fix cmake build failure with gflags. - Add CI tests for both gflags 2.1 and 2.2. - Fix ctest config with gtest. - Add CI to run test with ctest. One benefit of ctest is it support timeout, it's set to 5min in our CI, so we will know which test is hang. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8324 Test Plan: CI pass Reviewed By: ajkr Differential Revision: D28762517 Pulled By: jay-zhuang fbshipit-source-id: 09063c5af5f9f33abfcdeb48593acbd9826cd199
3 years ago · eda83eaac0
parent ab718b415f
commit eda83eaac0
5 changed files with 74 additions and 40 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -49,6 +49,9 @@ commands:
            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
            echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
+            echo "export GTEST_COLOR=1" >> $BASH_ENV
+            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
+            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV

  pre-steps-macos:
      steps:
@ -88,6 +91,14 @@ commands:
          command: |
            sudo apt-get update -y && sudo apt-get install -y libgflags-dev

+  upgrade-cmake:
+    steps:
+      - run:
+          name: Upgrade cmake
+          command: |
+            sudo apt remove --purge cmake
+            sudo snap install cmake --classic
+
  install-gflags-on-macos:
    steps:
      - run:
@ -140,12 +151,12 @@ jobs:
      - install-cmake-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
+      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) | .circleci/cat_ignore_eagain
      - post-steps

  build-linux:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -155,7 +166,7 @@ jobs:

  build-linux-mem-env:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -165,7 +176,7 @@ jobs:

  build-linux-encrypted-env:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -175,7 +186,7 @@ jobs:

  build-linux-shared_lib-alt_namespace-status_checked:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -185,7 +196,7 @@ jobs:

  build-linux-release:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
@ -212,7 +223,7 @@ jobs:

  build-linux-lite:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -222,7 +233,7 @@ jobs:

  build-linux-lite-release:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
@ -235,7 +246,7 @@ jobs:

  build-linux-clang-no_test_run:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: xlarge
    steps:
      - checkout # check out the code in the project directory
@ -245,7 +256,7 @@ jobs:

  build-linux-clang10-asan:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -256,7 +267,7 @@ jobs:

  build-linux-clang10-mini-tsan:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -267,7 +278,7 @@ jobs:

  build-linux-clang10-ubsan:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -278,7 +289,7 @@ jobs:

  build-linux-clang10-clang-analyze:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -288,13 +299,25 @@ jobs:
      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
      - post-steps

-  build-linux-cmake-no_test_run:
+  build-linux-cmake:
    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
+      image: ubuntu-1604:202104-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - install-gflags
+      - upgrade-cmake
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
+      - post-steps
+
+  build-linux-cmake-ubuntu-20:
+    machine:
+      image: ubuntu-2004:202104-01
+    resource_class: 2xlarge
    steps:
      - checkout # check out the code in the project directory
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain
+      - install-gflags
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-unity:
@ -309,7 +332,7 @@ jobs:

  build-linux-gcc-4_8-no_test_run:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - pre-steps
@ -417,7 +440,7 @@ jobs:

  build-linux-java:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    environment:
      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
@ -441,7 +464,7 @@ jobs:

  build-linux-java-static:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    environment:
      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
@ -512,7 +535,7 @@ jobs:

  build-examples:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - pre-steps
@ -525,7 +548,7 @@ jobs:

  build-cmake-mingw:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -548,7 +571,7 @@ jobs:

  build-linux-non-shm:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    parameters:
      start_test:
@ -604,6 +627,7 @@ jobs:
      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
    steps:
      - pre-steps
+      - install-gflags
      - run:
          name: "Set Java Environment"
          command: |
@ -616,7 +640,7 @@ jobs:
          command: |
            mkdir build
            cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
+            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
            make -j4
      - run:
          name: "Build Java with cmake"
@ -624,13 +648,13 @@ jobs:
            rm -rf build
            mkdir build
            cd build
-            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=0 ..
+            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
            make -j4 rocksdb rocksdbjni
      - post-steps

  build-format-compatible:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@ -650,6 +674,10 @@ workflows:
  build-linux:
    jobs:
      - build-linux
+  build-linux-cmake:
+    jobs:
+      - build-linux-cmake
+      - build-linux-cmake-ubuntu-20
  build-linux-mem-env:
    jobs:
      - build-linux-mem-env
@ -733,7 +761,6 @@ workflows:
  build-linux-compilers-no_test_run:
    jobs:
      - build-linux-clang-no_test_run
-      - build-linux-cmake-no_test_run
      - build-linux-gcc-4_8-no_test_run
      - build-linux-gcc-8-no_test_run
      - build-linux-gcc-9-no_test_run
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -32,10 +32,11 @@
 # 3. cmake ..
 # 4. make -j

-cmake_minimum_required(VERSION 3.5.1)
+cmake_minimum_required(VERSION 3.10)

 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
 include(ReadVersion)
+include(GoogleTest)
 get_rocksdb_version(rocksdb_VERSION)
 project(rocksdb
  VERSION ${rocksdb_VERSION}
@ -71,6 +72,12 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files,
 if (WITH_WINDOWS_UTF8_FILENAMES)
  add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
 endif()
+
+if ($ENV{CIRCLECI})
+  message(STATUS "Build for CircieCI env, a few tests may be disabled")
+  add_definitions(-DCIRCLECI)
+endif()
+
 # third-party/folly is only validated to work on Linux and Windows for now.
 # So only turn it on there by default.
 if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
@ -123,11 +130,11 @@ else()
        set(GFLAGS_LIB ${GFLAGS_TARGET})
      else()
        # Config with GFLAGS_LIBRARIES available since gflags 2.1.0
-        set(GFLAGS_LIB ${GFLAGS_LIBRARIES})
+        set(GFLAGS_LIB ${gflags_LIBRARIES})
      endif()
    else()
      find_package(gflags REQUIRED)
-        set(GFLAGS_LIB gflags::gflags)
+      set(GFLAGS_LIB gflags::gflags)
    endif()
    include_directories(${GFLAGS_INCLUDE_DIR})
    list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB})
@ -211,7 +218,7 @@ if(MSVC)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
 else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
  if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes")
@ -1321,21 +1328,21 @@ if(WITH_TESTS)

  foreach(sourcefile ${TESTS})
      get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-      set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
+      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+      set_target_properties(${exename}${ARTIFACT_SUFFIX}
        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
        OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
      )
-      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB})
+      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB})
      if(NOT "${exename}" MATCHES "db_sanity_test")
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
+        gtest_discover_tests(${exename})
+        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
      endif()
      if("${exename}" MATCHES "env_librados_test")
        # env_librados_test.cc uses librados directly
-        target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} rados)
+        target_link_libraries(${exename}${ARTIFACT_SUFFIX} rados)
      endif()
  endforeach(sourcefile ${TESTS})

--- a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
+++ b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
@ -9,7 +9,7 @@
 int main() {
  fprintf(stderr,
          "Please install gflags to run block_cache_trace_analyzer_test\n");
-  return 1;
+  return 0;
 }
 #else

--- a/tools/io_tracer_parser_test.cc
+++ b/tools/io_tracer_parser_test.cc
@ -9,7 +9,7 @@
 #include <cstdio>
 int main() {
  fprintf(stderr, "Please install gflags to run io_tracer_parser_test\n");
-  return 1;
+  return 0;
 }
 #else

--- a/tools/trace_analyzer_test.cc
+++ b/tools/trace_analyzer_test.cc
@ -12,7 +12,7 @@
 #include <cstdio>
 int main() {
  fprintf(stderr, "Please install gflags to run trace_analyzer test\n");
-  return 1;
+  return 0;
 }
 #else