Migrate away from Travis+Linux+amd64 (#7791)

Summary:
This disables Linux/amd64 builds in Travis for PRs, and adds a
gcc-10+c++20 build in CircleCI, which should fill out sufficient coverage
vs. what we had in Travis

Fixed a use of std::is_pod, which is deprecated in c++20

Fixed ++ on a volatile in db_repl_stress.cc, with bigger refactoring.
Although ++ on this volatile was probably ok with one thread writer and
one thread reader, the code was still overly complex. There was a
deadcode check for error
`if (replThread.no_read < dataPump.no_records)` which can be proven
never to happen based on the structure of the code. It infinite loops
instead for the case intended to be checked. I just simplified the code
for what should be the same checking power.

Also most configurations seem to be using make parallelism = 2 * vcores,
so fixing / using that.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7791

Test Plan:
CI
and `while ./db_repl_stress; do echo again; done` for a while

Reviewed By: siying

Differential Revision: D25669834

Pulled By: pdillinger

fbshipit-source-id: b2c688053d0b1d52c989903449d3cd27a04130d6

.circleci/config.yml

@@ -169,13 +169,13 @@ jobs:
   build-linux-release:
     machine:
       image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
+    resource_class: large
     steps:
       - checkout # check out the code in the project directory
-      - run: make V=1 -j32 release | .circleci/cat_ignore_eagain
+      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
       - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
       - install-gflags
-      - run: make V=1 -j32 release | .circleci/cat_ignore_eagain
+      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
       - run: ./db_stress --version # ensure with gflags
       - post-steps
 
@@ -186,11 +186,11 @@ jobs:
     steps:
       - checkout # check out the code in the project directory
       - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
       - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
       - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
       - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
       - run: ./db_stress --version # ensure with gflags
 
   build-linux-lite:
@@ -209,21 +209,21 @@ jobs:
     resource_class: large
     steps:
       - checkout # check out the code in the project directory
-      - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
+      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
       - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
       - install-gflags
-      - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
+      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
       - run: ./db_stress --version # ensure with gflags
       - post-steps
 
-  build-linux-clang-no-test:
+  build-linux-clang-no_test_run:
     machine:
       image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
+    resource_class: xlarge
     steps:
       - checkout # check out the code in the project directory
       - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
-      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain
+      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain
       - post-steps
 
   build-linux-clang10-asan:
@@ -271,33 +271,63 @@ jobs:
       - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
       - post-steps
 
-  build-linux-cmake:
+  build-linux-cmake-no_test_run:
     machine:
       image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
+    resource_class: large
     steps:
       - checkout # check out the code in the project directory
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain
       - post-steps
 
   build-linux-unity:
     docker: # executor type
       - image: gcc:latest
-    resource_class: xlarge
+    resource_class: large
     steps:
       - checkout # check out the code in the project directory
       - run: apt-get update -y && apt-get install -y libgflags-dev
-      - run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain
+      - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain
       - post-steps
 
-  build-linux-gcc-4-8:
+  build-linux-gcc-4_8-no_test_run:
     machine:
       image: ubuntu-1604:202007-01
     resource_class: large
     steps:
       - pre-steps
       - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
-      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - post-steps
+
+  build-linux-gcc-8-no_test_run:
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: large
+    steps:
+      - pre-steps
+      - run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
+      - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - post-steps
+
+  build-linux-gcc-9-no_test_run:
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: large
+    steps:
+      - pre-steps
+      - run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev
+      - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - post-steps
+
+  build-linux-gcc-10-cxx20-no_test_run:
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: xlarge
+    steps:
+      - pre-steps
+      - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
+      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
       - post-steps
 
   build-windows:
@@ -547,9 +577,6 @@ workflows:
   build-linux-lite-release:
     jobs:
       - build-linux-lite-release
-  build-linux-clang-no-test:
-    jobs:
-      - build-linux-clang-no-test
   build-linux-clang10-asan:
     jobs:
       - build-linux-clang10-asan
@@ -562,9 +589,6 @@ workflows:
   build-linux-clang10-clang-analyze:
     jobs:
       - build-linux-clang10-clang-analyze
-  build-linux-cmake:
-    jobs:
-      - build-linux-cmake
   build-linux-unity:
     jobs:
       - build-linux-unity
@@ -608,9 +632,14 @@ workflows:
       - build-linux-non-shm:
           start_test: "statistics_test" # make sure unique in src.mk
           end_test: ""
-  build-linux-gcc-4-8:
+  build-linux-compilers-no_test_run:
     jobs:
-      - build-linux-gcc-4-8
+      - build-linux-clang-no_test_run
+      - build-linux-cmake-no_test_run
+      - build-linux-gcc-4_8-no_test_run
+      - build-linux-gcc-8-no_test_run
+      - build-linux-gcc-9-no_test_run
+      - build-linux-gcc-10-cxx20-no_test_run
   build-macos:
     jobs:
       - build-macos

.travis.yml

@@ -86,19 +86,11 @@ matrix:
     env: JOB_NAME=make-gcc4.8
   - os: linux
     compiler: clang
-  # Exclude all but most unique cmake variants for pull requests, but build all in branches
+  # With migration to CircleCI, exclude Linux/amd64 for pull requests
+  # (but build in branches for now)
   - if: type = pull_request AND commit_message !~ /FULL_CI/
     os : linux
     arch: amd64
-    env: JOB_NAME=cmake
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: amd64
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: amd64
-    env: JOB_NAME=cmake-gcc9
   # Exclude most osx, arm64 and ppc64le tests for pull requests, but build in branches
   # Temporarily disable ppc64le cmake test while snapd is broken
   - if: type = pull_request AND commit_message !~ /FULL_CI/

build_tools/build_detect_platform

@@ -45,8 +45,13 @@ if test -z "$OUTPUT"; then
   exit 1
 fi
 
-# we depend on C++11
+# we depend on C++11, but should be compatible with newer standards
+if [ "$ROCKSDB_CXX_STANDARD" ]; then
+  PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD"
+else
   PLATFORM_CXXFLAGS="-std=c++11"
+fi
+
 # we currently depend on POSIX platform
 COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
 

table/block_based/filter_policy.cc

@@ -63,6 +63,8 @@ class XXH3pFilterBitsBuilder : public BuiltinFilterBitsBuilder {
   }
 
  protected:
+  static constexpr uint32_t kMetadataLen = 5;
+
   // For delegating between XXH3pFilterBitsBuilders
   void SwapEntriesWith(XXH3pFilterBitsBuilder* other) {
     std::swap(hash_entries_, other->hash_entries_);

tools/db_repl_stress.cc

@@ -27,13 +27,20 @@ int main() {
 // --num_inserts = the num of inserts the first thread should perform.
 // --wal_ttl = the wal ttl for the run.
 
+DEFINE_uint64(num_inserts, 1000,
+              "the num of inserts the first thread should"
+              " perform.");
+DEFINE_uint64(wal_ttl_seconds, 1000, "the wal ttl for the run(in seconds)");
+DEFINE_uint64(wal_size_limit_MB, 10,
+              "the wal size limit for the run"
+              "(in MB)");
+
 using namespace ROCKSDB_NAMESPACE;
 
 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
 using GFLAGS_NAMESPACE::SetUsageMessage;
 
 struct DataPumpThread {
-  size_t no_records;
   DB* db;  // Assumption DB is Open'ed already.
 };
 
@@ -41,8 +48,8 @@ static void DataPumpThreadBody(void* arg) {
   DataPumpThread* t = reinterpret_cast<DataPumpThread*>(arg);
   DB* db = t->db;
   Random rnd(301);
-  size_t i = 0;
-  while (i++ < t->no_records) {
+  uint64_t i = 0;
+  while (i++ < FLAGS_num_inserts) {
     if (!db->Put(WriteOptions(), Slice(rnd.RandomString(500)),
                  Slice(rnd.RandomString(500)))
              .ok()) {
@@ -52,45 +59,6 @@ static void DataPumpThreadBody(void* arg) {
   }
 }
 
-struct ReplicationThread {
-  std::atomic<bool> stop;
-  DB* db;
-  volatile size_t no_read;
-};
-
-static void ReplicationThreadBody(void* arg) {
-  ReplicationThread* t = reinterpret_cast<ReplicationThread*>(arg);
-  DB* db = t->db;
-  std::unique_ptr<TransactionLogIterator> iter;
-  SequenceNumber currentSeqNum = 1;
-  while (!t->stop.load(std::memory_order_acquire)) {
-    iter.reset();
-    Status s;
-    while (!db->GetUpdatesSince(currentSeqNum, &iter).ok()) {
-      if (t->stop.load(std::memory_order_acquire)) {
-        return;
-      }
-    }
-    fprintf(stderr, "Refreshing iterator\n");
-    for (; iter->Valid(); iter->Next(), t->no_read++, currentSeqNum++) {
-      BatchResult res = iter->GetBatch();
-      if (res.sequence != currentSeqNum) {
-        fprintf(stderr, "Missed a seq no. b/w %ld and %ld\n",
-                (long)currentSeqNum, (long)res.sequence);
-        exit(1);
-      }
-    }
-  }
-}
-
-DEFINE_uint64(num_inserts, 1000,
-              "the num of inserts the first thread should"
-              " perform.");
-DEFINE_uint64(wal_ttl_seconds, 1000, "the wal ttl for the run(in seconds)");
-DEFINE_uint64(wal_size_limit_MB, 10,
-              "the wal size limit for the run"
-              "(in MB)");
-
 int main(int argc, const char** argv) {
   SetUsageMessage(
       std::string("\nUSAGE:\n") + std::string(argv[0]) +
@@ -117,29 +85,38 @@ int main(int argc, const char** argv) {
   }
 
   DataPumpThread dataPump;
-  dataPump.no_records = FLAGS_num_inserts;
   dataPump.db = db;
   env->StartThread(DataPumpThreadBody, &dataPump);
 
-  ReplicationThread replThread;
-  replThread.db = db;
-  replThread.no_read = 0;
-  replThread.stop.store(false, std::memory_order_release);
-
-  env->StartThread(ReplicationThreadBody, &replThread);
-  while (replThread.no_read < FLAGS_num_inserts)
-    ;
-  replThread.stop.store(true, std::memory_order_release);
-  if (replThread.no_read < dataPump.no_records) {
-    // no. read should be => than inserted.
-    fprintf(stderr,
-            "No. of Record's written and read not same\nRead : %" ROCKSDB_PRIszt
-            " Written : %" ROCKSDB_PRIszt "\n",
-            replThread.no_read, dataPump.no_records);
+  std::unique_ptr<TransactionLogIterator> iter;
+  SequenceNumber currentSeqNum = 1;
+  uint64_t num_read = 0;
+  for (;;) {
+    iter.reset();
+    // Continue to probe a bit more after all received
+    size_t probes = 0;
+    while (!db->GetUpdatesSince(currentSeqNum, &iter).ok()) {
+      probes++;
+      if (probes > 100 && num_read >= FLAGS_num_inserts) {
+        if (num_read > FLAGS_num_inserts) {
+          fprintf(stderr, "Too many updates read: %ld expected: %ld\n",
+                  (long)num_read, (long)FLAGS_num_inserts);
           exit(1);
         }
         fprintf(stderr, "Successful!\n");
-  exit(0);
+        return 0;
+      }
+    }
+    fprintf(stderr, "Refreshing iterator\n");
+    for (; iter->Valid(); iter->Next(), num_read++, currentSeqNum++) {
+      BatchResult res = iter->GetBatch();
+      if (res.sequence != currentSeqNum) {
+        fprintf(stderr, "Missed a seq no. b/w %ld and %ld\n",
+                (long)currentSeqNum, (long)res.sequence);
+        exit(1);
+      }
+    }
+  }
 }
 
 #endif  // GFLAGS

utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h

@@ -29,7 +29,9 @@
 #define paranoid_invariant(a) assert(a)
 
 #define ENSURE_POD(type)                                                    \
-  static_assert(std::is_pod<type>::value, #type "isn't POD")
+  static_assert(                                                            \
+      std::is_standard_layout<type>::value && std::is_trivial<type>::value, \
+      #type "isn't POD")
 
 inline int get_error_errno(void) {
   invariant(errno);