From 538eec0661e6e66bc54fc189295c342a84ea7f54 Mon Sep 17 00:00:00 2001 From: Islam AbdelRahman Date: Fri, 22 Jan 2016 11:56:32 -0800 Subject: [PATCH] Update fbcode_config.sh to use latest versions automatically Summary: Update fbcode_config.sh so that It try to use the latest version for dependencies that we are using, after updating the code these libraries where updated ``` Snappy: 1.0.3 => 1.1.3 GFLAGS: 1.6 => 2.1.1 JEMALLOC: 3.6.0 => 4.0.3 ``` I have also updated clang from 3.7 to 3.7.1 ``` Clang 3.7 => 3.7.1 ``` Another change is that we use the same tp2 directory as fbcode, so we dont need to keep changing commit hash every time we need to change a version of a compiler or a library Test Plan: make check -j64 USE_CLANG=1 make check -j64 DISABLE_JEMALLOC=1 OPT=-g make all valgrind_check -j32 (make sure it's running) Reviewers: yhchiang, anthony, rven, kradhakrishnan, andrewkr, sdong Reviewed By: sdong Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D53037 --- build_tools/dependencies.sh | 13 +++++ build_tools/fbcode_config.sh | 69 +++++++++++++------------- build_tools/update_dependencies.sh | 79 ++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 35 deletions(-) create mode 100644 build_tools/dependencies.sh create mode 100755 build_tools/update_dependencies.sh diff --git a/build_tools/dependencies.sh b/build_tools/dependencies.sh new file mode 100644 index 000000000..a7e590904 --- /dev/null +++ b/build_tools/dependencies.sh @@ -0,0 +1,13 @@ +GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.9.x/centos6-native/* +CLANG_BASE=/mnt/vol/engshare/fbcode/third-party2/clang/3.7.1 +LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/5710d6a0fb0d12820aac0bffcd7fcb8646e7fff7/4.9.x/gcc-4.9-glibc-2.20/024dbc3 +GLIBC_BASE=/mnt/gvfs/third-party2/glibc/0600c95b31226b5e535614c590677d87c62d8016/2.20/gcc-4.9-glibc-2.20/500e281 +SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.9-glibc-2.20/e9936bf +ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.9-glibc-2.20/e9936bf +BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.9-glibc-2.20/e9936bf +LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf +ZSTD_BASE=/mnt/gvfs/third-party2/zstd/d4ac2c5f9be76d57a6cbd3eb1011e97574a56cde/0.4.5/gcc-4.9-glibc-2.20/e9936bf +GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a +JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/5f0be745ddc0f86f22c8c8bb64b6b1163c93df91/4.0.3/gcc-4.9-glibc-2.20/a6c5e1e +NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.9-glibc-2.20/e9936bf +LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/121f1a75c4414683aea8c70b761bfaf187f7c1a3/trunk/gcc-4.9-glibc-2.20/12266b1 diff --git a/build_tools/fbcode_config.sh b/build_tools/fbcode_config.sh index 48327c2e3..8cc0b53ff 100644 --- a/build_tools/fbcode_config.sh +++ b/build_tools/fbcode_config.sh @@ -6,85 +6,84 @@ # Environment variables that change the behavior of this script: # PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included + +source "$PWD/build_tools/dependencies.sh" + +TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2" + CFLAGS="" -# location of libgcc -LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/0473c80518a10d6efcbe24c5eeca3fb4ec9b519c/4.9.x/gcc-4.9-glibc-2.20/e1a7e4e" +# libgcc LIBGCC_INCLUDE="$LIBGCC_BASE/include" LIBGCC_LIBS=" -L $LIBGCC_BASE/libs" -# location of glibc -GLIBC_REV=7397bed99280af5d9543439cdb7d018af7542720 -GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/include" -GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/lib" - -SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/include/" +# glibc +GLIBC_INCLUDE="$GLIBC_BASE/include" +GLIBC_LIBS=" -L $GLIB_BASE/lib" +# snappy +SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" if test -z $PIC_BUILD; then - SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy.a" + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" else - SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy_pic.a" + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" fi - CFLAGS+=" -DSNAPPY" if test -z $PIC_BUILD; then # location of zlib headers and libraries - ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/include/" - ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/lib/libz.a" + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" CFLAGS+=" -DZLIB" # location of bzip headers and libraries - BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/include/" - BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/lib/libbz2.a" + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" CFLAGS+=" -DBZIP2" - LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/include/" - LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/lib/liblz4.a" + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" CFLAGS+=" -DLZ4" - ZSTD_REV=d4ac2c5f9be76d57a6cbd3eb1011e97574a56cde - ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.5/gcc-4.8.1-glibc-2.17/c3f970a/include" - ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.5/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a" + ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" CFLAGS+=" -DZSTD" fi # location of gflags headers and libraries -GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/include/" +GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" if test -z $PIC_BUILD; then - GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags.a" + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" else - GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags_pic.a" + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a" fi CFLAGS+=" -DGFLAGS=google" # location of jemalloc -JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/include/" -JEMALLOC_LIB=" /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/lib/libjemalloc.a" +JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" +JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a" if test -z $PIC_BUILD; then # location of numa - NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/include/" - NUMA_LIB=" /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/lib/libnuma.a" + NUMA_INCLUDE=" -I $NUMA_BASE/include/" + NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" CFLAGS+=" -DNUMA" # location of libunwind - LIBUNWIND="/mnt/gvfs/third-party2/libunwind/1de3b75e0afedfe5585b231bbb340ec7a1542335/1.1/gcc-4.9-glibc-2.20/34235e8/lib/libunwind.a" + LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" fi # use Intel SSE support for checksum calculations export USE_SSE=1 -BINUTILS="/mnt/gvfs/third-party2/binutils/0b6ad0c88ddd903333a48ae8bff134efac468e4a/2.25/centos6-native/da39a3e/bin" +BINUTILS="$TP2_LATEST/binutils/2.25/centos6-native/*/bin" AR="$BINUTILS/ar" DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE" -GCC_BASE="/mnt/gvfs/third-party2/gcc/1c67a0b88f64d4d9ced0382d141c76aaa7d62fba/4.9.x/centos6-native/1317bc4" STDLIBS="-L $GCC_BASE/lib64" -CLANG_BASE="/mnt/gvfs/third-party2/clang/d81444dd214df3d2466734de45bb264a0486acc3/dev" -CLANG_BIN="$CLANG_BASE/centos6-native/af4b1a0/bin" +CLANG_BIN="$CLANG_BASE/centos6-native/*/bin" CLANG_ANALYZER="$CLANG_BIN/clang++" CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build" @@ -98,11 +97,11 @@ if [ -z "$USE_CLANG" ]; then CFLAGS+=" -isystem $LIBGCC_INCLUDE" else # clang - CLANG_INCLUDE="$CLANG_BASE/gcc-4.9-glibc-2.20/74c386f/lib/clang/dev/include/" + CLANG_INCLUDE="$CLANG_BASE/centos6-native/*/lib/clang/*/include" CC="$CLANG_BIN/clang" CXX="$CLANG_BIN/clang++" - KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/ffd14f660a43c4b92717986b1bba66722ef089d0/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/da39a3e/include" + KERNEL_HEADERS_INCLUDE="$TP2_LATEST/kernel-headers/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/*/include" CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib" CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x " @@ -128,6 +127,6 @@ PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS" -VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/6c45ef049cbf11c2df593addb712cd891049e737/3.10.0/gcc-4.9-glibc-2.20/4230243/bin/" +VALGRIND_VER="$TP2_LATEST/valgrind/3.10.0/gcc-4.9-glibc-2.20/*/bin/" export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD diff --git a/build_tools/update_dependencies.sh b/build_tools/update_dependencies.sh new file mode 100755 index 000000000..6a9c662a2 --- /dev/null +++ b/build_tools/update_dependencies.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# +# Update dependencies.sh file with the latest avaliable versions + +BASEDIR=$(dirname $0) +OUTPUT="$BASEDIR/dependencies.sh" + +rm -f "$OUTPUT" +touch "$OUTPUT" + +function log_variable() +{ + echo "$1=${!1}" >> "$OUTPUT" +} + + +TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2" +## $1 => lib name +## $2 => lib version (if not provided, will try to pick latest) +## $3 => platform (if not provided, will try to pick latest gcc) +## +## get_lib_base will set a variable named ${LIB_NAME}_BASE to the lib location +function get_lib_base() +{ + local lib_name=$1 + local lib_version=$2 + local lib_platform=$3 + + local result="$TP2_LATEST/$lib_name/" + + # Lib Version + if [ -z "$lib_version" ]; then + # version is not provided, use latest + result=`ls -dr1v $result/*/ | head -n1` + else + result="$result/$lib_version/" + fi + + # Lib Platform + if [ -z "$lib_platform" ]; then + # platform is not provided, use latest gcc + result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1` + else + result="$result/$lib_platform/" + fi + + result="$result/*/" + + # lib_name => LIB_NAME_BASE + local __res_var=${lib_name^^}"_BASE" + # LIB_NAME_BASE=$result + eval $__res_var=`readlink -f $result` + + log_variable $__res_var +} + +echo "Writing dependencies to $OUTPUT" + +# Compilers locations +GCC_BASE="$TP2_LATEST/gcc/4.9.x/centos6-native/*" +CLANG_BASE="$TP2_LATEST/clang/3.7.1" + +log_variable GCC_BASE +log_variable CLANG_BASE + +# Libraries locations +get_lib_base libgcc +get_lib_base glibc 2.20 gcc-4.9-glibc-2.20 +get_lib_base snappy +get_lib_base zlib +get_lib_base bzip2 +get_lib_base lz4 +get_lib_base zstd +get_lib_base gflags +get_lib_base jemalloc +get_lib_base numa +get_lib_base libunwind + +git diff $OUTPUT