From 4d897e51dfe1abfd9b5e9f11db1efd26265b4ba3 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Tue, 22 Dec 2020 00:19:44 -0800 Subject: [PATCH] Migrate away from Travis+Linux+amd64 (#7791) Summary: This disables Linux/amd64 builds in Travis for PRs, and adds a gcc-10+c++20 build in CircleCI, which should fill out sufficient coverage vs. what we had in Travis Fixed a use of std::is_pod, which is deprecated in c++20 Fixed ++ on a volatile in db_repl_stress.cc, with bigger refactoring. Although ++ on this volatile was probably ok with one thread writer and one thread reader, the code was still overly complex. There was a deadcode check for error `if (replThread.no_read < dataPump.no_records)` which can be proven never to happen based on the structure of the code. It infinite loops instead for the case intended to be checked. I just simplified the code for what should be the same checking power. Also most configurations seem to be using make parallelism = 2 * vcores, so fixing / using that. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7791 Test Plan: CI and `while ./db_repl_stress; do echo again; done` for a while Reviewed By: siying Differential Revision: D25669834 Pulled By: pdillinger fbshipit-source-id: b2c688053d0b1d52c989903449d3cd27a04130d6 --- .circleci/config.yml | 79 ++++++++++----- .travis.yml | 12 +-- build_tools/build_detect_platform | 9 +- table/block_based/filter_policy.cc | 2 + tools/db_repl_stress.cc | 99 +++++++------------ .../lib/portability/toku_assert_subst.h | 6 +- 6 files changed, 107 insertions(+), 100 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e56088aa..ab309146c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -169,13 +169,13 @@ jobs: build-linux-release: machine: image: ubuntu-1604:202007-01 - resource_class: 2xlarge + resource_class: large steps: - checkout # check out the code in the project directory - - run: make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: make V=1 -j8 release | .circleci/cat_ignore_eagain - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - install-gflags - - run: make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: make V=1 -j8 release | .circleci/cat_ignore_eagain - run: ./db_stress --version # ensure with gflags - post-steps @@ -186,11 +186,11 @@ jobs: steps: - checkout # check out the code in the project directory - run: make clean - - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev - run: make clean - - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain - run: ./db_stress --version # ensure with gflags build-linux-lite: @@ -209,21 +209,21 @@ jobs: resource_class: large steps: - checkout # check out the code in the project directory - - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - install-gflags - - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain + - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain - run: ./db_stress --version # ensure with gflags - post-steps - build-linux-clang-no-test: + build-linux-clang-no_test_run: machine: image: ubuntu-1604:202007-01 - resource_class: 2xlarge + resource_class: xlarge steps: - checkout # check out the code in the project directory - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev - - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain - post-steps build-linux-clang10-asan: @@ -271,33 +271,63 @@ jobs: - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. - post-steps - build-linux-cmake: + build-linux-cmake-no_test_run: machine: image: ubuntu-1604:202007-01 - resource_class: 2xlarge + resource_class: large steps: - checkout # check out the code in the project directory - - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain + - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain - post-steps build-linux-unity: docker: # executor type - image: gcc:latest - resource_class: xlarge + resource_class: large steps: - checkout # check out the code in the project directory - run: apt-get update -y && apt-get install -y libgflags-dev - - run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain + - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain - post-steps - build-linux-gcc-4-8: + build-linux-gcc-4_8-no_test_run: machine: image: ubuntu-1604:202007-01 resource_class: large steps: - pre-steps - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev - - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI + - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI + - post-steps + + build-linux-gcc-8-no_test_run: + machine: + image: ubuntu-2004:202010-01 + resource_class: large + steps: + - pre-steps + - run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev + - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI + - post-steps + + build-linux-gcc-9-no_test_run: + machine: + image: ubuntu-2004:202010-01 + resource_class: large + steps: + - pre-steps + - run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev + - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI + - post-steps + + build-linux-gcc-10-cxx20-no_test_run: + machine: + image: ubuntu-2004:202010-01 + resource_class: xlarge + steps: + - pre-steps + - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev + - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI - post-steps build-windows: @@ -547,9 +577,6 @@ workflows: build-linux-lite-release: jobs: - build-linux-lite-release - build-linux-clang-no-test: - jobs: - - build-linux-clang-no-test build-linux-clang10-asan: jobs: - build-linux-clang10-asan @@ -562,9 +589,6 @@ workflows: build-linux-clang10-clang-analyze: jobs: - build-linux-clang10-clang-analyze - build-linux-cmake: - jobs: - - build-linux-cmake build-linux-unity: jobs: - build-linux-unity @@ -608,9 +632,14 @@ workflows: - build-linux-non-shm: start_test: "statistics_test" # make sure unique in src.mk end_test: "" - build-linux-gcc-4-8: + build-linux-compilers-no_test_run: jobs: - - build-linux-gcc-4-8 + - build-linux-clang-no_test_run + - build-linux-cmake-no_test_run + - build-linux-gcc-4_8-no_test_run + - build-linux-gcc-8-no_test_run + - build-linux-gcc-9-no_test_run + - build-linux-gcc-10-cxx20-no_test_run build-macos: jobs: - build-macos diff --git a/.travis.yml b/.travis.yml index 51604a86e..2acff576e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,19 +86,11 @@ matrix: env: JOB_NAME=make-gcc4.8 - os: linux compiler: clang - # Exclude all but most unique cmake variants for pull requests, but build all in branches + # With migration to CircleCI, exclude Linux/amd64 for pull requests + # (but build in branches for now) - if: type = pull_request AND commit_message !~ /FULL_CI/ os : linux arch: amd64 - env: JOB_NAME=cmake - - if: type = pull_request AND commit_message !~ /FULL_CI/ - os : linux - arch: amd64 - env: JOB_NAME=cmake-gcc8 - - if: type = pull_request AND commit_message !~ /FULL_CI/ - os : linux - arch: amd64 - env: JOB_NAME=cmake-gcc9 # Exclude most osx, arm64 and ppc64le tests for pull requests, but build in branches # Temporarily disable ppc64le cmake test while snapd is broken - if: type = pull_request AND commit_message !~ /FULL_CI/ diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index a686ab06d..9ee81e661 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -45,8 +45,13 @@ if test -z "$OUTPUT"; then exit 1 fi -# we depend on C++11 -PLATFORM_CXXFLAGS="-std=c++11" +# we depend on C++11, but should be compatible with newer standards +if [ "$ROCKSDB_CXX_STANDARD" ]; then + PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD" +else + PLATFORM_CXXFLAGS="-std=c++11" +fi + # we currently depend on POSIX platform COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX" diff --git a/table/block_based/filter_policy.cc b/table/block_based/filter_policy.cc index 0c26de816..664aadd18 100644 --- a/table/block_based/filter_policy.cc +++ b/table/block_based/filter_policy.cc @@ -63,6 +63,8 @@ class XXH3pFilterBitsBuilder : public BuiltinFilterBitsBuilder { } protected: + static constexpr uint32_t kMetadataLen = 5; + // For delegating between XXH3pFilterBitsBuilders void SwapEntriesWith(XXH3pFilterBitsBuilder* other) { std::swap(hash_entries_, other->hash_entries_); diff --git a/tools/db_repl_stress.cc b/tools/db_repl_stress.cc index 794ac1530..7025e1040 100644 --- a/tools/db_repl_stress.cc +++ b/tools/db_repl_stress.cc @@ -27,13 +27,20 @@ int main() { // --num_inserts = the num of inserts the first thread should perform. // --wal_ttl = the wal ttl for the run. +DEFINE_uint64(num_inserts, 1000, + "the num of inserts the first thread should" + " perform."); +DEFINE_uint64(wal_ttl_seconds, 1000, "the wal ttl for the run(in seconds)"); +DEFINE_uint64(wal_size_limit_MB, 10, + "the wal size limit for the run" + "(in MB)"); + using namespace ROCKSDB_NAMESPACE; using GFLAGS_NAMESPACE::ParseCommandLineFlags; using GFLAGS_NAMESPACE::SetUsageMessage; struct DataPumpThread { - size_t no_records; DB* db; // Assumption DB is Open'ed already. }; @@ -41,8 +48,8 @@ static void DataPumpThreadBody(void* arg) { DataPumpThread* t = reinterpret_cast(arg); DB* db = t->db; Random rnd(301); - size_t i = 0; - while (i++ < t->no_records) { + uint64_t i = 0; + while (i++ < FLAGS_num_inserts) { if (!db->Put(WriteOptions(), Slice(rnd.RandomString(500)), Slice(rnd.RandomString(500))) .ok()) { @@ -52,45 +59,6 @@ static void DataPumpThreadBody(void* arg) { } } -struct ReplicationThread { - std::atomic stop; - DB* db; - volatile size_t no_read; -}; - -static void ReplicationThreadBody(void* arg) { - ReplicationThread* t = reinterpret_cast(arg); - DB* db = t->db; - std::unique_ptr iter; - SequenceNumber currentSeqNum = 1; - while (!t->stop.load(std::memory_order_acquire)) { - iter.reset(); - Status s; - while (!db->GetUpdatesSince(currentSeqNum, &iter).ok()) { - if (t->stop.load(std::memory_order_acquire)) { - return; - } - } - fprintf(stderr, "Refreshing iterator\n"); - for (; iter->Valid(); iter->Next(), t->no_read++, currentSeqNum++) { - BatchResult res = iter->GetBatch(); - if (res.sequence != currentSeqNum) { - fprintf(stderr, "Missed a seq no. b/w %ld and %ld\n", - (long)currentSeqNum, (long)res.sequence); - exit(1); - } - } - } -} - -DEFINE_uint64(num_inserts, 1000, - "the num of inserts the first thread should" - " perform."); -DEFINE_uint64(wal_ttl_seconds, 1000, "the wal ttl for the run(in seconds)"); -DEFINE_uint64(wal_size_limit_MB, 10, - "the wal size limit for the run" - "(in MB)"); - int main(int argc, const char** argv) { SetUsageMessage( std::string("\nUSAGE:\n") + std::string(argv[0]) + @@ -117,29 +85,38 @@ int main(int argc, const char** argv) { } DataPumpThread dataPump; - dataPump.no_records = FLAGS_num_inserts; dataPump.db = db; env->StartThread(DataPumpThreadBody, &dataPump); - ReplicationThread replThread; - replThread.db = db; - replThread.no_read = 0; - replThread.stop.store(false, std::memory_order_release); - - env->StartThread(ReplicationThreadBody, &replThread); - while (replThread.no_read < FLAGS_num_inserts) - ; - replThread.stop.store(true, std::memory_order_release); - if (replThread.no_read < dataPump.no_records) { - // no. read should be => than inserted. - fprintf(stderr, - "No. of Record's written and read not same\nRead : %" ROCKSDB_PRIszt - " Written : %" ROCKSDB_PRIszt "\n", - replThread.no_read, dataPump.no_records); - exit(1); + std::unique_ptr iter; + SequenceNumber currentSeqNum = 1; + uint64_t num_read = 0; + for (;;) { + iter.reset(); + // Continue to probe a bit more after all received + size_t probes = 0; + while (!db->GetUpdatesSince(currentSeqNum, &iter).ok()) { + probes++; + if (probes > 100 && num_read >= FLAGS_num_inserts) { + if (num_read > FLAGS_num_inserts) { + fprintf(stderr, "Too many updates read: %ld expected: %ld\n", + (long)num_read, (long)FLAGS_num_inserts); + exit(1); + } + fprintf(stderr, "Successful!\n"); + return 0; + } + } + fprintf(stderr, "Refreshing iterator\n"); + for (; iter->Valid(); iter->Next(), num_read++, currentSeqNum++) { + BatchResult res = iter->GetBatch(); + if (res.sequence != currentSeqNum) { + fprintf(stderr, "Missed a seq no. b/w %ld and %ld\n", + (long)currentSeqNum, (long)res.sequence); + exit(1); + } + } } - fprintf(stderr, "Successful!\n"); - exit(0); } #endif // GFLAGS diff --git a/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h b/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h index bfc2c810e..af47800fb 100644 --- a/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h +++ b/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h @@ -28,8 +28,10 @@ #define paranoid_invariant_notnull(a) assert(a) #define paranoid_invariant(a) assert(a) -#define ENSURE_POD(type) \ - static_assert(std::is_pod::value, #type "isn't POD") +#define ENSURE_POD(type) \ + static_assert( \ + std::is_standard_layout::value && std::is_trivial::value, \ + #type "isn't POD") inline int get_error_errno(void) { invariant(errno);