Update fbcode_config.sh to use latest versions automatically

Summary:
Update fbcode_config.sh so that It try to use the latest version for dependencies that we are using, after updating the code these libraries where updated

```
Snappy: 1.0.3 => 1.1.3
GFLAGS: 1.6 => 2.1.1
JEMALLOC: 3.6.0 => 4.0.3
```

I have also updated clang from 3.7 to 3.7.1

```
Clang 3.7 => 3.7.1
```

Another change is that we use the same tp2 directory as fbcode, so we dont need to keep changing commit hash every time we need to change a version of a compiler or a library

Test Plan:
make check -j64
USE_CLANG=1 make check -j64

DISABLE_JEMALLOC=1 OPT=-g make all valgrind_check -j32 (make sure it's running)

Reviewers: yhchiang, anthony, rven, kradhakrishnan, andrewkr, sdong

Reviewed By: sdong

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D53037
main
Islam AbdelRahman 9 years ago
parent 8019aa9b55
commit 538eec0661
  1. 13
      build_tools/dependencies.sh
  2. 69
      build_tools/fbcode_config.sh
  3. 79
      build_tools/update_dependencies.sh

@ -0,0 +1,13 @@
GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.9.x/centos6-native/*
CLANG_BASE=/mnt/vol/engshare/fbcode/third-party2/clang/3.7.1
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/5710d6a0fb0d12820aac0bffcd7fcb8646e7fff7/4.9.x/gcc-4.9-glibc-2.20/024dbc3
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/0600c95b31226b5e535614c590677d87c62d8016/2.20/gcc-4.9-glibc-2.20/500e281
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.9-glibc-2.20/e9936bf
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.9-glibc-2.20/e9936bf
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.9-glibc-2.20/e9936bf
LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/d4ac2c5f9be76d57a6cbd3eb1011e97574a56cde/0.4.5/gcc-4.9-glibc-2.20/e9936bf
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/5f0be745ddc0f86f22c8c8bb64b6b1163c93df91/4.0.3/gcc-4.9-glibc-2.20/a6c5e1e
NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.9-glibc-2.20/e9936bf
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/121f1a75c4414683aea8c70b761bfaf187f7c1a3/trunk/gcc-4.9-glibc-2.20/12266b1

@ -6,85 +6,84 @@
# Environment variables that change the behavior of this script: # Environment variables that change the behavior of this script:
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included # PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
source "$PWD/build_tools/dependencies.sh"
TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2"
CFLAGS="" CFLAGS=""
# location of libgcc # libgcc
LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/0473c80518a10d6efcbe24c5eeca3fb4ec9b519c/4.9.x/gcc-4.9-glibc-2.20/e1a7e4e"
LIBGCC_INCLUDE="$LIBGCC_BASE/include" LIBGCC_INCLUDE="$LIBGCC_BASE/include"
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs" LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
# location of glibc # glibc
GLIBC_REV=7397bed99280af5d9543439cdb7d018af7542720 GLIBC_INCLUDE="$GLIBC_BASE/include"
GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/include" GLIBC_LIBS=" -L $GLIB_BASE/lib"
GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/lib"
SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/include/"
# snappy
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
if test -z $PIC_BUILD; then if test -z $PIC_BUILD; then
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy.a" SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
else else
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy_pic.a" SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
fi fi
CFLAGS+=" -DSNAPPY" CFLAGS+=" -DSNAPPY"
if test -z $PIC_BUILD; then if test -z $PIC_BUILD; then
# location of zlib headers and libraries # location of zlib headers and libraries
ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/include/" ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/lib/libz.a" ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
CFLAGS+=" -DZLIB" CFLAGS+=" -DZLIB"
# location of bzip headers and libraries # location of bzip headers and libraries
BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/include/" BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/lib/libbz2.a" BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
CFLAGS+=" -DBZIP2" CFLAGS+=" -DBZIP2"
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/include/" LZ4_INCLUDE=" -I $LZ4_BASE/include/"
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/lib/liblz4.a" LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
CFLAGS+=" -DLZ4" CFLAGS+=" -DLZ4"
ZSTD_REV=d4ac2c5f9be76d57a6cbd3eb1011e97574a56cde ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.5/gcc-4.8.1-glibc-2.17/c3f970a/include" ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.5/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a"
CFLAGS+=" -DZSTD" CFLAGS+=" -DZSTD"
fi fi
# location of gflags headers and libraries # location of gflags headers and libraries
GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/include/" GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
if test -z $PIC_BUILD; then if test -z $PIC_BUILD; then
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags.a" GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
else else
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags_pic.a" GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
fi fi
CFLAGS+=" -DGFLAGS=google" CFLAGS+=" -DGFLAGS=google"
# location of jemalloc # location of jemalloc
JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/include/" JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
JEMALLOC_LIB=" /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/lib/libjemalloc.a" JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
if test -z $PIC_BUILD; then if test -z $PIC_BUILD; then
# location of numa # location of numa
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/include/" NUMA_INCLUDE=" -I $NUMA_BASE/include/"
NUMA_LIB=" /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/lib/libnuma.a" NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
CFLAGS+=" -DNUMA" CFLAGS+=" -DNUMA"
# location of libunwind # location of libunwind
LIBUNWIND="/mnt/gvfs/third-party2/libunwind/1de3b75e0afedfe5585b231bbb340ec7a1542335/1.1/gcc-4.9-glibc-2.20/34235e8/lib/libunwind.a" LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
fi fi
# use Intel SSE support for checksum calculations # use Intel SSE support for checksum calculations
export USE_SSE=1 export USE_SSE=1
BINUTILS="/mnt/gvfs/third-party2/binutils/0b6ad0c88ddd903333a48ae8bff134efac468e4a/2.25/centos6-native/da39a3e/bin" BINUTILS="$TP2_LATEST/binutils/2.25/centos6-native/*/bin"
AR="$BINUTILS/ar" AR="$BINUTILS/ar"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE" DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
GCC_BASE="/mnt/gvfs/third-party2/gcc/1c67a0b88f64d4d9ced0382d141c76aaa7d62fba/4.9.x/centos6-native/1317bc4"
STDLIBS="-L $GCC_BASE/lib64" STDLIBS="-L $GCC_BASE/lib64"
CLANG_BASE="/mnt/gvfs/third-party2/clang/d81444dd214df3d2466734de45bb264a0486acc3/dev" CLANG_BIN="$CLANG_BASE/centos6-native/*/bin"
CLANG_BIN="$CLANG_BASE/centos6-native/af4b1a0/bin"
CLANG_ANALYZER="$CLANG_BIN/clang++" CLANG_ANALYZER="$CLANG_BIN/clang++"
CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build" CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build"
@ -98,11 +97,11 @@ if [ -z "$USE_CLANG" ]; then
CFLAGS+=" -isystem $LIBGCC_INCLUDE" CFLAGS+=" -isystem $LIBGCC_INCLUDE"
else else
# clang # clang
CLANG_INCLUDE="$CLANG_BASE/gcc-4.9-glibc-2.20/74c386f/lib/clang/dev/include/" CLANG_INCLUDE="$CLANG_BASE/centos6-native/*/lib/clang/*/include"
CC="$CLANG_BIN/clang" CC="$CLANG_BIN/clang"
CXX="$CLANG_BIN/clang++" CXX="$CLANG_BIN/clang++"
KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/ffd14f660a43c4b92717986b1bba66722ef089d0/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/da39a3e/include" KERNEL_HEADERS_INCLUDE="$TP2_LATEST/kernel-headers/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/*/include"
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib" CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x " CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
@ -128,6 +127,6 @@ PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS" EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/6c45ef049cbf11c2df593addb712cd891049e737/3.10.0/gcc-4.9-glibc-2.20/4230243/bin/" VALGRIND_VER="$TP2_LATEST/valgrind/3.10.0/gcc-4.9-glibc-2.20/*/bin/"
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD

@ -0,0 +1,79 @@
#!/bin/sh
#
# Update dependencies.sh file with the latest avaliable versions
BASEDIR=$(dirname $0)
OUTPUT="$BASEDIR/dependencies.sh"
rm -f "$OUTPUT"
touch "$OUTPUT"
function log_variable()
{
echo "$1=${!1}" >> "$OUTPUT"
}
TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2"
## $1 => lib name
## $2 => lib version (if not provided, will try to pick latest)
## $3 => platform (if not provided, will try to pick latest gcc)
##
## get_lib_base will set a variable named ${LIB_NAME}_BASE to the lib location
function get_lib_base()
{
local lib_name=$1
local lib_version=$2
local lib_platform=$3
local result="$TP2_LATEST/$lib_name/"
# Lib Version
if [ -z "$lib_version" ]; then
# version is not provided, use latest
result=`ls -dr1v $result/*/ | head -n1`
else
result="$result/$lib_version/"
fi
# Lib Platform
if [ -z "$lib_platform" ]; then
# platform is not provided, use latest gcc
result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
else
result="$result/$lib_platform/"
fi
result="$result/*/"
# lib_name => LIB_NAME_BASE
local __res_var=${lib_name^^}"_BASE"
# LIB_NAME_BASE=$result
eval $__res_var=`readlink -f $result`
log_variable $__res_var
}
echo "Writing dependencies to $OUTPUT"
# Compilers locations
GCC_BASE="$TP2_LATEST/gcc/4.9.x/centos6-native/*"
CLANG_BASE="$TP2_LATEST/clang/3.7.1"
log_variable GCC_BASE
log_variable CLANG_BASE
# Libraries locations
get_lib_base libgcc
get_lib_base glibc 2.20 gcc-4.9-glibc-2.20
get_lib_base snappy
get_lib_base zlib
get_lib_base bzip2
get_lib_base lz4
get_lib_base zstd
get_lib_base gflags
get_lib_base jemalloc
get_lib_base numa
get_lib_base libunwind
git diff $OUTPUT
Loading…
Cancel
Save