From d5dfa8c6fe74d8810b85d5cae1fd98a6b0427bb8 Mon Sep 17 00:00:00 2001 From: sdong Date: Fri, 15 Apr 2022 16:05:38 -0700 Subject: [PATCH] Upgrade development environment. (#9843) Summary: It's to support Meta's internal environment platform010. Gcc still doesn't work but USE_CLANG=1 should work. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9843 Test Plan: Try to make and ROCKSDB_FBCODE_BUILD_WITH_PLATFORM010=1 USE_CLANG=1 make Reviewed By: pdillinger Differential Revision: D35652507 fbshipit-source-id: a4a14b2fa4a2d6ca6fbf1b65060e81c39f079363 --- Makefile | 2 +- build_tools/build_detect_platform | 8 +- build_tools/dependencies_platform010.sh | 22 ++++ build_tools/fbcode_config_platform010.sh | 156 +++++++++++++++++++++++ build_tools/update_dependencies.sh | 131 +++++++------------ 5 files changed, 235 insertions(+), 84 deletions(-) create mode 100644 build_tools/dependencies_platform010.sh create mode 100644 build_tools/fbcode_config_platform010.sh diff --git a/Makefile b/Makefile index 380deb0b3..b81e5cf58 100644 --- a/Makefile +++ b/Makefile @@ -288,7 +288,7 @@ missing_make_config_paths := $(shell \ grep "\./\S*\|/\S*" -o $(CURDIR)/make_config.mk | \ while read path; \ do [ -e $$path ] || echo $$path; \ - done | sort | uniq) + done | sort | uniq | grep -v "/DOES/NOT/EXIST") $(foreach path, $(missing_make_config_paths), \ $(warning Warning: $(path) does not exist)) diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index fc8577979..0c53c416b 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -63,7 +63,13 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then if [ "$LIB_MODE" == "shared" ]; then PIC_BUILD=1 fi - source "$PWD/build_tools/fbcode_config_platform009.sh" + if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM010" ]; then + source "$PWD/build_tools/fbcode_config_platform010.sh" + elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then + source "$PWD/build_tools/fbcode_config_platform009.sh" + else + source "$PWD/build_tools/fbcode_config_platform009.sh" + fi fi # Delete existing output, if it exists diff --git a/build_tools/dependencies_platform010.sh b/build_tools/dependencies_platform010.sh new file mode 100644 index 000000000..38bd346d4 --- /dev/null +++ b/build_tools/dependencies_platform010.sh @@ -0,0 +1,22 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# The file is generated using update_dependencies.sh. +GCC_BASE=/mnt/gvfs/third-party2/gcc/e40bde78650fa91b8405a857e3f10bf336633fb0/11.x/centos7-native/886b5eb +CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/2043340983c032915adbb6f78903dc855b65aee8/12/platform010/9520e0f +LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c00dcc6a3e4125c7e8b248e9a79c14b78ac9e0ca/11.x/platform010/5684a5a +GLIBC_BASE=/mnt/gvfs/third-party2/glibc/0b9c8e4b060eda62f3bc1c6127bbe1256697569b/2.34/platform010/f259413 +SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/bc9647f7912b131315827d65cb6189c21f381d05/1.1.3/platform010/76ebdda +ZLIB_BASE=/mnt/gvfs/third-party2/zlib/a6f5f3f1d063d2d00cd02fc12f0f05fc3ab3a994/1.2.11/platform010/76ebdda +BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/09703139cfc376bd8a82642385a0e97726b28287/1.0.6/platform010/76ebdda +LZ4_BASE=/mnt/gvfs/third-party2/lz4/60220d6a5bf7722b9cc239a1368c596619b12060/1.9.1/platform010/76ebdda +ZSTD_BASE=/mnt/gvfs/third-party2/zstd/50eace8143eaaea9473deae1f3283e0049e05633/1.4.x/platform010/64091f4 +GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/5d27e5919771603da06000a027b12f799e58a4f7/2.2.0/platform010/76ebdda +JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/b62912d333ef33f9760efa6219dbe3fe6abb3b0e/master/platform010/f57cc4a +NUMA_BASE=/mnt/gvfs/third-party2/numa/6b412770957aa3c8a87e5e0dcd8cc2f45f393bc0/2.0.11/platform010/76ebdda +LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/52f69816e936e147664ad717eb71a1a0e9dc973a/1.4/platform010/5074a48 +TBB_BASE=/mnt/gvfs/third-party2/tbb/c9cc192099fa84c0dcd0ffeedd44a373ad6e4925/2018_U5/platform010/76ebdda +LIBURING_BASE=/mnt/gvfs/third-party2/liburing/a98e2d137007e3ebf7f33bd6f99c2c56bdaf8488/20210212/platform010/76ebdda +BENCHMARK_BASE=/mnt/gvfs/third-party2/benchmark/780c7a0f9cf0967961e69ad08e61cddd85d61821/trunk/platform010/76ebdda +KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/02d9f76aaaba580611cf75e741753c800c7fdc12/fb/platform010/da39a3e +BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/938dc3f064ef3a48c0446f5b11d788d50b3eb5ee/2.37/centos7-native/da39a3e +VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/429a6b3203eb415f1599bd15183659153129188e/3.15.0/platform010/76ebdda +LUA_BASE=/mnt/gvfs/third-party2/lua/363787fa5cac2a8aa20638909210443278fa138e/5.3.4/platform010/9079c97 diff --git a/build_tools/fbcode_config_platform010.sh b/build_tools/fbcode_config_platform010.sh new file mode 100644 index 000000000..346544f56 --- /dev/null +++ b/build_tools/fbcode_config_platform010.sh @@ -0,0 +1,156 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# +# Set environment variables so that we can compile rocksdb using +# fbcode settings. It uses the latest g++ and clang compilers and also +# uses jemalloc +# Environment variables that change the behavior of this script: +# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included + + +BASEDIR=`dirname $BASH_SOURCE` +source "$BASEDIR/dependencies_platform010.sh" + +# Disallow using libraries from default locations as they might not be compatible with platform010 libraries. +CFLAGS=" --sysroot=/DOES/NOT/EXIST" + +# libgcc +LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/trunk" +LIBGCC_LIBS=" -L $LIBGCC_BASE/lib -B$LIBGCC_BASE/lib/gcc/x86_64-facebook-linux/trunk/" + +# glibc +GLIBC_INCLUDE="$GLIBC_BASE/include" +GLIBC_LIBS=" -L $GLIBC_BASE/lib" +GLIBC_LIBS+=" -B$GLIBC_BASE/lib" + +if test -z $PIC_BUILD; then + MAYBE_PIC= +else + MAYBE_PIC=_pic +fi + +# snappy +SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" +SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a" +CFLAGS+=" -DSNAPPY" + +# location of zlib headers and libraries +ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" +ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a" +CFLAGS+=" -DZLIB" + +# location of bzip headers and libraries +BZIP_INCLUDE=" -I $BZIP2_BASE/include/" +BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a" +CFLAGS+=" -DBZIP2" + +LZ4_INCLUDE=" -I $LZ4_BASE/include/" +LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a" +CFLAGS+=" -DLZ4" + +ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" +ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a" +CFLAGS+=" -DZSTD" + +# location of gflags headers and libraries +GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" +GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags${MAYBE_PIC}.a" +CFLAGS+=" -DGFLAGS=gflags" + +BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/" +BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark${MAYBE_PIC}.a" + +# location of jemalloc +JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" +JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc${MAYBE_PIC}.a" + +# location of numa +NUMA_INCLUDE=" -I $NUMA_BASE/include/" +NUMA_LIB=" $NUMA_BASE/lib/libnuma${MAYBE_PIC}.a" +CFLAGS+=" -DNUMA" + +# location of libunwind +LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind${MAYBE_PIC}.a" + +# location of TBB +TBB_INCLUDE=" -isystem $TBB_BASE/include/" +TBB_LIBS="$TBB_BASE/lib/libtbb${MAYBE_PIC}.a" +CFLAGS+=" -DTBB" + +# location of LIBURING +LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/" +LIBURING_LIBS="$LIBURING_BASE/lib/liburing${MAYBE_PIC}.a" +CFLAGS+=" -DLIBURING" + +test "$USE_SSE" || USE_SSE=1 +export USE_SSE +test "$PORTABLE" || PORTABLE=1 +export PORTABLE + +BINUTILS="$BINUTILS_BASE/bin" +AR="$BINUTILS/ar" +AS="$BINUTILS/as" + +DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE $BENCHMARK_INCLUDE" + +STDLIBS="-L $GCC_BASE/lib64" + +CLANG_BIN="$CLANG_BASE/bin" +CLANG_LIB="$CLANG_BASE/lib" +CLANG_SRC="$CLANG_BASE/../../src" + +CLANG_ANALYZER="$CLANG_BIN/clang++" +CLANG_SCAN_BUILD="$CLANG_SRC/llvm/clang/tools/scan-build/bin/scan-build" + +if [ -z "$USE_CLANG" ]; then + # gcc + CC="$GCC_BASE/bin/gcc" + CXX="$GCC_BASE/bin/g++" + AR="$GCC_BASE/bin/gcc-ar" + + + CFLAGS+=" -B$BINUTILS" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $GLIBC_INCLUDE" + JEMALLOC=1 +else + # clang + CLANG_INCLUDE="$CLANG_LIB/clang/stable/include" + CC="$CLANG_BIN/clang" + CXX="$CLANG_BIN/clang++" + AR="$CLANG_BIN/llvm-ar" + + CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib" + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/trunk " + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/trunk/x86_64-facebook-linux " + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $CLANG_INCLUDE" + CFLAGS+=" -Wno-expansion-to-defined " + CXXFLAGS="-nostdinc++" +fi + +KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include" +CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " +CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " + +CFLAGS+=" $DEPS_INCLUDE" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT" +CXXFLAGS+=" $CFLAGS" + +EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS" +EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform010/lib/ld.so" +EXEC_LDFLAGS+=" $LIBUNWIND" +EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform010/lib" +EXEC_LDFLAGS+=" -Wl,-rpath=$GCC_BASE/lib64" +# required by libtbb +EXEC_LDFLAGS+=" -ldl" + +PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" +PLATFORM_LDFLAGS+=" -B$BINUTILS" + +EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS" + +VALGRIND_VER="$VALGRIND_BASE/bin/" + +export CC CXX AR AS CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB diff --git a/build_tools/update_dependencies.sh b/build_tools/update_dependencies.sh index 99099a406..a2fdcd0ee 100755 --- a/build_tools/update_dependencies.sh +++ b/build_tools/update_dependencies.sh @@ -9,6 +9,7 @@ OUTPUT="" function log_header() { echo "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved." >> "$OUTPUT" + echo "# The file is generated using update_dependencies.sh." >> "$OUTPUT" } @@ -18,7 +19,7 @@ function log_variable() } -TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2" +TP2_LATEST="/data/users/$USER/fbsource/fbcode/third-party2/" ## $1 => lib name ## $2 => lib version (if not provided, will try to pick latest) ## $3 => platform (if not provided, will try to pick latest gcc) @@ -50,6 +51,8 @@ function get_lib_base() fi result=`ls -1d $result/*/ | head -n1` + + echo Finding link $result # lib_name => LIB_NAME_BASE local __res_var=${lib_name^^}"_BASE" @@ -61,10 +64,10 @@ function get_lib_base() } ########################################################### -# platform007 dependencies # +# platform010 dependencies # ########################################################### -OUTPUT="$BASEDIR/dependencies_platform007.sh" +OUTPUT="$BASEDIR/dependencies_platform010.sh" rm -f "$OUTPUT" touch "$OUTPUT" @@ -72,40 +75,42 @@ touch "$OUTPUT" echo "Writing dependencies to $OUTPUT" # Compilers locations -GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/` -CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/` +GCC_BASE=`readlink -f $TP2_LATEST/gcc/11.x/centos7-native/*/` +CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/12/platform010/*/` log_header log_variable GCC_BASE log_variable CLANG_BASE # Libraries locations -get_lib_base libgcc 7.x platform007 -get_lib_base glibc 2.26 platform007 -get_lib_base snappy LATEST platform007 -get_lib_base zlib LATEST platform007 -get_lib_base bzip2 LATEST platform007 -get_lib_base lz4 LATEST platform007 -get_lib_base zstd LATEST platform007 -get_lib_base gflags LATEST platform007 -get_lib_base jemalloc LATEST platform007 -get_lib_base numa LATEST platform007 -get_lib_base libunwind LATEST platform007 -get_lib_base tbb LATEST platform007 -get_lib_base liburing LATEST platform007 - -get_lib_base kernel-headers fb platform007 +get_lib_base libgcc 11.x platform010 +get_lib_base glibc 2.34 platform010 +get_lib_base snappy LATEST platform010 +get_lib_base zlib LATEST platform010 +get_lib_base bzip2 LATEST platform010 +get_lib_base lz4 LATEST platform010 +get_lib_base zstd LATEST platform010 +get_lib_base gflags LATEST platform010 +get_lib_base jemalloc LATEST platform010 +get_lib_base numa LATEST platform010 +get_lib_base libunwind LATEST platform010 +get_lib_base tbb 2018_U5 platform010 +get_lib_base liburing LATEST platform010 +get_lib_base benchmark LATEST platform010 + +get_lib_base kernel-headers fb platform010 get_lib_base binutils LATEST centos7-native -get_lib_base valgrind LATEST platform007 -get_lib_base lua 5.3.4 platform007 +get_lib_base valgrind LATEST platform010 +get_lib_base lua 5.3.4 platform010 git diff $OUTPUT + ########################################################### -# 5.x dependencies # +# platform009 dependencies # ########################################################### -OUTPUT="$BASEDIR/dependencies.sh" +OUTPUT="$BASEDIR/dependencies_platform009.sh" rm -f "$OUTPUT" touch "$OUTPUT" @@ -113,70 +118,32 @@ touch "$OUTPUT" echo "Writing dependencies to $OUTPUT" # Compilers locations -GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos7-native/*/` -CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/` +GCC_BASE=`readlink -f $TP2_LATEST/gcc/9.x/centos7-native/*/` +CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/9.0.0/platform009/*/` log_header log_variable GCC_BASE log_variable CLANG_BASE # Libraries locations -get_lib_base libgcc 5.x gcc-5-glibc-2.23 -get_lib_base glibc 2.23 gcc-5-glibc-2.23 -get_lib_base snappy LATEST gcc-5-glibc-2.23 -get_lib_base zlib LATEST gcc-5-glibc-2.23 -get_lib_base bzip2 LATEST gcc-5-glibc-2.23 -get_lib_base lz4 LATEST gcc-5-glibc-2.23 -get_lib_base zstd LATEST gcc-5-glibc-2.23 -get_lib_base gflags LATEST gcc-5-glibc-2.23 -get_lib_base jemalloc LATEST gcc-5-glibc-2.23 -get_lib_base numa LATEST gcc-5-glibc-2.23 -get_lib_base libunwind LATEST gcc-5-glibc-2.23 -get_lib_base tbb LATEST gcc-5-glibc-2.23 - -get_lib_base kernel-headers 4.0.9-36_fbk5_2933_gd092e3f gcc-5-glibc-2.23 +get_lib_base libgcc 9.x platform009 +get_lib_base glibc 2.30 platform009 +get_lib_base snappy LATEST platform009 +get_lib_base zlib LATEST platform009 +get_lib_base bzip2 LATEST platform009 +get_lib_base lz4 LATEST platform009 +get_lib_base zstd LATEST platform009 +get_lib_base gflags LATEST platform009 +get_lib_base jemalloc LATEST platform009 +get_lib_base numa LATEST platform009 +get_lib_base libunwind LATEST platform009 +get_lib_base tbb 2018_U5 platform009 +get_lib_base liburing LATEST platform009 +get_lib_base benchmark LATEST platform009 + +get_lib_base kernel-headers fb platform009 get_lib_base binutils LATEST centos7-native -get_lib_base valgrind LATEST gcc-5-glibc-2.23 -get_lib_base lua 5.2.3 gcc-5-glibc-2.23 - -git diff $OUTPUT - -########################################################### -# 4.8.1 dependencies # -########################################################### - -OUTPUT="$BASEDIR/dependencies_4.8.1.sh" - -rm -f "$OUTPUT" -touch "$OUTPUT" - -echo "Writing 4.8.1 dependencies to $OUTPUT" - -# Compilers locations -GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/` -CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/` - -log_header -log_variable GCC_BASE -log_variable CLANG_BASE - -# Libraries locations -get_lib_base libgcc 4.8.1 gcc-4.8.1-glibc-2.17 -get_lib_base glibc 2.17 gcc-4.8.1-glibc-2.17 -get_lib_base snappy LATEST gcc-4.8.1-glibc-2.17 -get_lib_base zlib LATEST gcc-4.8.1-glibc-2.17 -get_lib_base bzip2 LATEST gcc-4.8.1-glibc-2.17 -get_lib_base lz4 LATEST gcc-4.8.1-glibc-2.17 -get_lib_base zstd LATEST gcc-4.8.1-glibc-2.17 -get_lib_base gflags LATEST gcc-4.8.1-glibc-2.17 -get_lib_base jemalloc LATEST gcc-4.8.1-glibc-2.17 -get_lib_base numa LATEST gcc-4.8.1-glibc-2.17 -get_lib_base libunwind LATEST gcc-4.8.1-glibc-2.17 -get_lib_base tbb 4.0_update2 gcc-4.8.1-glibc-2.17 - -get_lib_base kernel-headers LATEST gcc-4.8.1-glibc-2.17 -get_lib_base binutils LATEST centos6-native -get_lib_base valgrind 3.8.1 gcc-4.8.1-glibc-2.17 -get_lib_base lua 5.2.3 centos6-native +get_lib_base valgrind LATEST platform009 +get_lib_base lua 5.3.4 platform009 git diff $OUTPUT