From e7358da9a201f5524d911ecea7d4a3f83ff1f59d Mon Sep 17 00:00:00 2001 From: sdong Date: Wed, 12 Aug 2020 19:28:28 -0700 Subject: [PATCH] Upgrade tool chain (#7251) Summary: Upgrade tool chain to the latest. It is done mostly manually as build_tools/build_detect_platform fails to update many of them. Try to fix a new clang analyze warning with the new tool chain. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7251 Test Plan: "make all", "USE_CLANG=1 make all" Reviewed By: riversand963 Differential Revision: D23091090 fbshipit-source-id: 732e5a30137837431438f85f36296406b641f975 --- build_tools/build_detect_platform | 4 +- build_tools/dependencies_platform009.sh | 20 +++ build_tools/fbcode_config_platform009.sh | 172 +++++++++++++++++++++++ db/corruption_test.cc | 5 + 4 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 build_tools/dependencies_platform009.sh create mode 100644 build_tools/fbcode_config_platform009.sh diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 66f83b2b0..e76cb9cd2 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -60,8 +60,10 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then source "$PWD/build_tools/fbcode_config4.8.1.sh" elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then source "$PWD/build_tools/fbcode_config.sh" - else + elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then source "$PWD/build_tools/fbcode_config_platform007.sh" + else + source "$PWD/build_tools/fbcode_config_platform009.sh" fi fi diff --git a/build_tools/dependencies_platform009.sh b/build_tools/dependencies_platform009.sh new file mode 100644 index 000000000..37a56d3ab --- /dev/null +++ b/build_tools/dependencies_platform009.sh @@ -0,0 +1,20 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +GCC_BASE=/mnt/gvfs/third-party2/gcc/1795efe5f06778c15a92c8f9a2aba5dc496d9d4d/9.x/centos7-native/3bed279 +CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/7318eaac22659b6ff2fe43918e4b69fd0772a8a7/9.0.0/platform009/651ee30 +LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/4959b39cfbe5965a37c861c4c327fa7c5c759b87/9.x/platform009/9202ce7 +GLIBC_BASE=/mnt/gvfs/third-party2/glibc/45ce3375cdc77ecb2520bbf8f0ecddd3f98efd7a/2.30/platform009/f259413 +SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/be4de3205e029101b18aa8103daa696c2bef3b19/1.1.3/platform009/7f3b187 +ZLIB_BASE=/mnt/gvfs/third-party2/zlib/3c160ac5c67e257501e24c6c1d00ad5e01d73db6/1.2.8/platform009/7f3b187 +BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/73a237ac5bc0a5f5d67b39b8d253cfebaab88684/1.0.6/platform009/7f3b187 +LZ4_BASE=/mnt/gvfs/third-party2/lz4/ec6573523b0ce55ef6373a4801189027cf07bb2c/1.9.1/platform009/7f3b187 +ZSTD_BASE=/mnt/gvfs/third-party2/zstd/64c58a207d2495e83abc57a500a956df09b79a7c/1.4.x/platform009/ba86d1f +GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/824d0a8a5abb5b121afd1b35fc3896407ea50092/2.2.0/platform009/7f3b187 +JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9aef9feb850b168a68736420f217b01cce11a89/master/platform009/c305944 +NUMA_BASE=/mnt/gvfs/third-party2/numa/0af65f71e23a67bf65dc91b11f95caa39325c432/2.0.11/platform009/7f3b187 +LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/02486dac347645d31dce116f44e1de3177315be2/1.4/platform009/5191652 +TBB_BASE=/mnt/gvfs/third-party2/tbb/2e0ec671e550bfca347300bf3f789d9c0fff24ad/2018_U5/platform009/7f3b187 +LIBURING_BASE=/mnt/gvfs/third-party2/liburing/70dbd9cfee63a25611417d09433a86d7711b3990/20200729/platform009/7f3b187 +KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/32b8a2407b634df3f8f948ba373fc4acc6a18296/fb/platform009/da39a3e +BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/08634589372fa5f237bfd374e8c644a8364e78c1/2.32/centos7-native/da39a3e +VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/6ae525939ad02e5e676855082fbbc7828dbafeac/3.15.0/platform009/7f3b187 +LUA_BASE=/mnt/gvfs/third-party2/lua/162efd9561a3d21f6869f4814011e9cf1b3ff4dc/5.3.4/platform009/a6271c4 diff --git a/build_tools/fbcode_config_platform009.sh b/build_tools/fbcode_config_platform009.sh new file mode 100644 index 000000000..cf24bd815 --- /dev/null +++ b/build_tools/fbcode_config_platform009.sh @@ -0,0 +1,172 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# +# Set environment variables so that we can compile rocksdb using +# fbcode settings. It uses the latest g++ and clang compilers and also +# uses jemalloc +# Environment variables that change the behavior of this script: +# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included + + +BASEDIR=`dirname $BASH_SOURCE` +source "$BASEDIR/dependencies_platform009.sh" + +CFLAGS="" + +# libgcc +LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/9.3.0" +LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" + +# glibc +GLIBC_INCLUDE="$GLIBC_BASE/include" +GLIBC_LIBS=" -L $GLIBC_BASE/lib" + +# snappy +SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" +if test -z $PIC_BUILD; then + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" +else + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" +fi +CFLAGS+=" -DSNAPPY" + +if test -z $PIC_BUILD; then + # location of zlib headers and libraries + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" + CFLAGS+=" -DZLIB" + + # location of bzip headers and libraries + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" + CFLAGS+=" -DBZIP2" + + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" + CFLAGS+=" -DLZ4" +fi + +ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" +if test -z $PIC_BUILD; then + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" +else + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a" +fi +CFLAGS+=" -DZSTD" + +# location of gflags headers and libraries +GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" +if test -z $PIC_BUILD; then + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" +else + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a" +fi +CFLAGS+=" -DGFLAGS=gflags" + +# location of jemalloc +JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" +JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a" + +if test -z $PIC_BUILD; then + # location of numa + NUMA_INCLUDE=" -I $NUMA_BASE/include/" + NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" + CFLAGS+=" -DNUMA" + + # location of libunwind + LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" +fi + +# location of TBB +TBB_INCLUDE=" -isystem $TBB_BASE/include/" +if test -z $PIC_BUILD; then + TBB_LIBS="$TBB_BASE/lib/libtbb.a" +else + TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a" +fi +CFLAGS+=" -DTBB" + +# location of LIBURING +LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/" +if test -z $PIC_BUILD; then + LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a" +else + LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a" +fi +CFLAGS+=" -DLIBURING" + +test "$USE_SSE" || USE_SSE=1 +export USE_SSE +test "$PORTABLE" || PORTABLE=1 +export PORTABLE + +BINUTILS="$BINUTILS_BASE/bin" +AR="$BINUTILS/ar" + +DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE" + +STDLIBS="-L $GCC_BASE/lib64" + +CLANG_BIN="$CLANG_BASE/bin" +CLANG_LIB="$CLANG_BASE/lib" +CLANG_SRC="$CLANG_BASE/../../src" + +CLANG_ANALYZER="$CLANG_BIN/clang++" +CLANG_SCAN_BUILD="$CLANG_SRC/llvm/clang/tools/scan-build/bin/scan-build" + +if [ -z "$USE_CLANG" ]; then + # gcc + CC="$GCC_BASE/bin/gcc" + CXX="$GCC_BASE/bin/g++" + AR="$GCC_BASE/bin/gcc-ar" + + CFLAGS+=" -B$BINUTILS/gold" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $GLIBC_INCLUDE" + JEMALLOC=1 +else + # clang + CLANG_INCLUDE="$CLANG_LIB/clang/stable/include" + CC="$CLANG_BIN/clang" + CXX="$CLANG_BIN/clang++" + AR="$CLANG_BIN/llvm-ar" + + KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include" + + CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib" + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x " + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x/x86_64-facebook-linux " + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $CLANG_INCLUDE" + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " + CFLAGS+=" -Wno-expansion-to-defined " + CXXFLAGS="-nostdinc++" +fi + +CFLAGS+=" $DEPS_INCLUDE" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT" +CXXFLAGS+=" $CFLAGS" + +EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS" +EXEC_LDFLAGS+=" -B$BINUTILS/gold" +EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform009/lib/ld.so" +EXEC_LDFLAGS+=" $LIBUNWIND" +EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform009/lib" +# required by libtbb +EXEC_LDFLAGS+=" -ldl" + +PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" + +EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS" + +VALGRIND_VER="$VALGRIND_BASE/bin/" + +# lua not supported because it's on track for deprecation, I think +LUA_PATH= +LUA_LIB= + +USE_FOLLY_DISTRIBUTED_MUTEX=1 + +export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB diff --git a/db/corruption_test.cc b/db/corruption_test.cc index ede373752..1adff85b9 100644 --- a/db/corruption_test.cc +++ b/db/corruption_test.cc @@ -69,6 +69,7 @@ class CorruptionTest : public testing::Test { ~CorruptionTest() override { delete db_; + db_ = nullptr; DestroyDB(dbname_, Options()); } @@ -572,12 +573,14 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) { Status s; for (const auto& mode : corruption_modes) { delete db_; + db_ = nullptr; s = DestroyDB(dbname_, options); std::shared_ptr mock = std::make_shared(); options.table_factory = mock; mock->SetCorruptionMode(mode); ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); Build(10); s = db_->Flush(FlushOptions()); if (mode == mock::MockTableFactory::kCorruptNone) { @@ -595,11 +598,13 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) { Status s; for (const auto& mode : corruption_modes) { delete db_; + db_ = nullptr; s = DestroyDB(dbname_, options); std::shared_ptr mock = std::make_shared(); options.table_factory = mock; ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); Build(100, 2); // ASSERT_OK(db_->Flush(FlushOptions())); DBImpl* dbi = static_cast_with_check(db_);