|
|
|
#!/usr/bin/env bash
|
|
|
|
#
|
|
|
|
# Detects OS we're compiling on and outputs a file specified by the first
|
|
|
|
# argument, which in turn gets read while processing Makefile.
|
|
|
|
#
|
|
|
|
# The output will set the following variables:
|
|
|
|
# CC C Compiler path
|
|
|
|
# CXX C++ Compiler path
|
|
|
|
# PLATFORM_LDFLAGS Linker flags
|
|
|
|
# JAVA_LDFLAGS Linker flags for RocksDBJava
|
|
|
|
# JAVA_STATIC_LDFLAGS Linker flags for RocksDBJava static build
|
|
|
|
# PLATFORM_SHARED_EXT Extension for shared libraries
|
|
|
|
# PLATFORM_SHARED_LDFLAGS Flags for building shared library
|
|
|
|
# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
|
|
|
|
# PLATFORM_CCFLAGS C compiler flags
|
|
|
|
# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
|
|
|
|
# PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned
|
|
|
|
# shared libraries, empty otherwise.
|
|
|
|
# FIND Command for the find utility
|
|
|
|
# WATCH Command for the watch utility
|
|
|
|
#
|
|
|
|
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
|
|
|
|
#
|
|
|
|
# -DROCKSDB_PLATFORM_POSIX if posix-platform based
|
|
|
|
# -DSNAPPY if the Snappy library is present
|
|
|
|
# -DLZ4 if the LZ4 library is present
|
|
|
|
# -DZSTD if the ZSTD library is present
|
|
|
|
# -DNUMA if the NUMA library is present
|
|
|
|
# -DTBB if the TBB library is present
|
|
|
|
#
|
|
|
|
# Using gflags in rocksdb:
|
|
|
|
# Our project depends on gflags, which requires users to take some extra steps
|
|
|
|
# before they can compile the whole repository:
|
|
|
|
# 1. Install gflags. You may download it from here:
|
|
|
|
# https://gflags.github.io/gflags/ (Mac users can `brew install gflags`)
|
|
|
|
# 2. Once installed, add the include path for gflags to your CPATH env var and
|
|
|
|
# the lib path to LIBRARY_PATH. If installed with default settings, the lib
|
|
|
|
# will be /usr/local/lib and the include path will be /usr/local/include
|
|
|
|
|
|
|
|
OUTPUT=$1
|
|
|
|
if test -z "$OUTPUT"; then
|
|
|
|
echo "usage: $0 <output-filename>" >&2
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
# we depend on C++11
|
|
|
|
PLATFORM_CXXFLAGS="-std=c++11"
|
|
|
|
# we currently depend on POSIX platform
|
|
|
|
COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
|
|
|
|
|
|
|
|
# Default to fbcode gcc on internal fb machines
|
|
|
|
if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
|
|
|
FBCODE_BUILD="true"
|
|
|
|
# If we're compiling with TSAN we need pic build
|
|
|
|
PIC_BUILD=$COMPILE_WITH_TSAN
|
|
|
|
if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
|
|
|
# we need this to build with MySQL. Don't use for other purposes.
|
|
|
|
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
|
|
|
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then
|
|
|
|
source "$PWD/build_tools/fbcode_config_platform007.sh"
|
|
|
|
else
|
|
|
|
source "$PWD/build_tools/fbcode_config.sh"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
# Delete existing output, if it exists
|
|
|
|
rm -f "$OUTPUT"
|
|
|
|
touch "$OUTPUT"
|
|
|
|
|
|
|
|
if test -z "$CC"; then
|
|
|
|
if [ -x "$(command -v cc)" ]; then
|
|
|
|
CC=cc
|
|
|
|
elif [ -x "$(command -v clang)" ]; then
|
|
|
|
CC=clang
|
|
|
|
else
|
|
|
|
CC=cc
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test -z "$CXX"; then
|
|
|
|
if [ -x "$(command -v g++)" ]; then
|
|
|
|
CXX=g++
|
|
|
|
elif [ -x "$(command -v clang++)" ]; then
|
|
|
|
CXX=clang++
|
|
|
|
else
|
|
|
|
CXX=g++
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
# Detect OS
|
|
|
|
if test -z "$TARGET_OS"; then
|
|
|
|
TARGET_OS=`uname -s`
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test -z "$TARGET_ARCHITECTURE"; then
|
|
|
|
TARGET_ARCHITECTURE=`uname -m`
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test -z "$CLANG_SCAN_BUILD"; then
|
|
|
|
CLANG_SCAN_BUILD=scan-build
|
|
|
|
fi
|
|
|
|
|
build: do not relink every single binary just for a timestamp
Summary:
Prior to this change, "make check" would always waste a lot of
time relinking 60+ binaries. With this change, it does that
only when the generated file, util/build_version.cc, changes,
and that happens only when the date changes or when the
current git SHA changes.
This change makes some other improvements: before, there was no
rule to build a deleted util/build_version.cc. If it was somehow
removed, any attempt to link a program would fail.
There is no longer any need for the separate file,
build_tools/build_detect_version. Its functionality is
now in the Makefile.
* Makefile (DEPFILES): Don't filter-out util/build_version.cc.
No need, and besides, removing that dependency was wrong.
(date, git_sha, gen_build_version): New helper variables.
(util/build_version.cc): New rule, to create this file
and update it only if it would contain new information.
* build_tools/build_detect_platform: Remove file.
* db/db_impl.cc: Now, print only date (not the time).
* util/build_version.h (rocksdb_build_compile_time): Remove
declaration. No longer used.
Test Plan:
- Run "make check" twice, and note that the second time no linking is performed.
- Remove util/build_version.cc and ensure that any "make"
command regenerates it before doing anything else.
- Run this: strings librocksdb.a|grep _build_.
That prints output including the following:
rocksdb_build_git_date:2015-02-19
rocksdb_build_git_sha:2.8.fb-1792-g3cb6cc0
Reviewers: ljin, sdong, igor
Reviewed By: igor
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D33591
10 years ago
|
|
|
if test -z "$CLANG_ANALYZER"; then
|
|
|
|
CLANG_ANALYZER=$(command -v clang++ 2> /dev/null)
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test -z "$FIND"; then
|
|
|
|
FIND=find
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test -z "$WATCH"; then
|
|
|
|
WATCH=watch
|
|
|
|
fi
|
|
|
|
|
fPIC in x64 environment
Summary:
Check https://github.com/facebook/rocksdb/pull/15 for context.
Apparently [1], we need -fPIC in x64 environments (this is added only in non-fbcode).
In fbcode, I removed -fPIC per @dhruba's suggestion, since it introduces perf regression. I'm not sure what would are the implications of doing that, but looks like it works, and when releasing to the third-party, we're disabling -fPIC either way [2].
Would love a suggestion from someone who knows more about this
[1] http://eli.thegreenplace.net/2011/11/11/position-independent-code-pic-in-shared-libraries-on-x64/
[2] https://our.intern.facebook.com/intern/wiki/index.php/Database/RocksDB/Third_Party
Test Plan: make check works
Reviewers: dhruba, emayanke, kailiu
Reviewed By: dhruba
CC: leveldb, dhruba, reconnect.grayhat
Differential Revision: https://reviews.facebook.net/D14337
11 years ago
|
|
|
COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}"
|
|
|
|
CROSS_COMPILE=
|
|
|
|
PLATFORM_CCFLAGS=
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS"
|
|
|
|
PLATFORM_SHARED_EXT="so"
|
|
|
|
PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl,"
|
|
|
|
PLATFORM_SHARED_CFLAGS="-fPIC"
|
|
|
|
PLATFORM_SHARED_VERSIONED=true
|
|
|
|
|
|
|
|
# generic port files (working on all platform by #ifdef) go directly in /port
|
|
|
|
GENERIC_PORT_FILES=`cd "$ROCKSDB_ROOT"; find port -name '*.cc' | tr "\n" " "`
|
|
|
|
|
|
|
|
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
|
|
|
|
case "$TARGET_OS" in
|
|
|
|
Darwin)
|
|
|
|
PLATFORM=OS_MACOSX
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX"
|
|
|
|
PLATFORM_SHARED_EXT=dylib
|
|
|
|
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
|
|
|
|
# PORT_FILES=port/darwin/darwin_specific.cc
|
|
|
|
;;
|
|
|
|
IOS)
|
|
|
|
PLATFORM=IOS
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE -DROCKSDB_LITE"
|
|
|
|
PLATFORM_SHARED_EXT=dylib
|
|
|
|
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
|
|
|
|
CROSS_COMPILE=true
|
|
|
|
PLATFORM_SHARED_VERSIONED=
|
|
|
|
;;
|
|
|
|
Linux)
|
|
|
|
PLATFORM=OS_LINUX
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
|
|
|
|
if [ -z "$USE_CLANG" ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
|
|
|
|
else
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
|
|
|
|
fi
|
[RocksDB] Added nano second stopwatch and new perf counters to track block read cost
Summary: The pupose of this diff is to expose per user-call level precise timing of block read, so that we can answer questions like: a Get() costs me 100ms, is that somehow related to loading blocks from file system, or sth else? We will answer that with EXACTLY how many blocks have been read, how much time was spent on transfering the bytes from os, how much time was spent on checksum verification and how much time was spent on block decompression, just for that one Get. A nano second stopwatch was introduced to track time with higher precision. The cost/precision of the stopwatch is also measured in unit-test. On my dev box, retrieving one time instance costs about 30ns, on average. The deviation of timing results is good enough to track 100ns-1us level events. And the overhead could be safely ignored for 100us level events (10000 instances/s), for example, a viewstate thrift call.
Test Plan: perf_context_test, also testing with viewstate shadow traffic.
Reviewers: dhruba
Reviewed By: dhruba
CC: leveldb, xjin
Differential Revision: https://reviews.facebook.net/D12351
12 years ago
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
|
|
|
|
# PORT_FILES=port/linux/linux_specific.cc
|
|
|
|
;;
|
|
|
|
SunOS)
|
|
|
|
PLATFORM=OS_SOLARIS
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
|
|
|
|
# PORT_FILES=port/sunos/sunos_specific.cc
|
|
|
|
;;
|
|
|
|
AIX)
|
|
|
|
PLATFORM=OS_AIX
|
|
|
|
CC=gcc
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
|
|
|
|
# PORT_FILES=port/aix/aix_specific.cc
|
|
|
|
;;
|
|
|
|
FreeBSD)
|
|
|
|
PLATFORM=OS_FREEBSD
|
|
|
|
CXX=clang++
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
|
|
|
|
# PORT_FILES=port/freebsd/freebsd_specific.cc
|
|
|
|
;;
|
|
|
|
NetBSD)
|
|
|
|
PLATFORM=OS_NETBSD
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s"
|
|
|
|
# PORT_FILES=port/netbsd/netbsd_specific.cc
|
|
|
|
;;
|
|
|
|
OpenBSD)
|
|
|
|
PLATFORM=OS_OPENBSD
|
|
|
|
CXX=clang++
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
|
|
|
|
# PORT_FILES=port/openbsd/openbsd_specific.cc
|
|
|
|
FIND=gfind
|
|
|
|
WATCH=gnuwatch
|
|
|
|
;;
|
|
|
|
DragonFly)
|
|
|
|
PLATFORM=OS_DRAGONFLYBSD
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
|
|
|
|
# PORT_FILES=port/dragonfly/dragonfly_specific.cc
|
|
|
|
;;
|
|
|
|
Cygwin)
|
|
|
|
PLATFORM=CYGWIN
|
|
|
|
PLATFORM_SHARED_CFLAGS=""
|
|
|
|
PLATFORM_CXXFLAGS="-std=gnu++11"
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN"
|
|
|
|
if [ -z "$USE_CLANG" ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
|
|
|
|
else
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
|
|
|
|
fi
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
|
|
|
|
# PORT_FILES=port/linux/linux_specific.cc
|
|
|
|
;;
|
|
|
|
OS_ANDROID_CROSSCOMPILE)
|
|
|
|
PLATFORM=OS_ANDROID
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DROCKSDB_PLATFORM_POSIX"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS " # All pthread features are in the Android C library
|
|
|
|
# PORT_FILES=port/android/android.cc
|
|
|
|
CROSS_COMPILE=true
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
echo "Unknown platform!" >&2
|
|
|
|
exit 1
|
|
|
|
esac
|
|
|
|
|
|
|
|
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
|
|
|
|
JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
|
|
|
|
JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
|
|
|
|
|
|
|
|
if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
|
|
|
|
# Cross-compiling; do not try any compilation tests.
|
|
|
|
# Also don't need any compilation tests if compiling on fbcode
|
|
|
|
true
|
|
|
|
else
|
|
|
|
if ! test $ROCKSDB_DISABLE_FALLOCATE; then
|
|
|
|
# Test whether fallocate is available
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <linux/falloc.h>
|
|
|
|
int main() {
|
|
|
|
int fd = open("/dev/null", 0);
|
|
|
|
fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_SNAPPY; then
|
|
|
|
# Test whether Snappy library is installed
|
|
|
|
# http://code.google.com/p/snappy/
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <snappy.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_GFLAGS; then
|
|
|
|
# Test whether gflags library is installed
|
|
|
|
# http://gflags.github.io/gflags/
|
|
|
|
# check if the namespace is gflags
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
|
|
|
|
#include <gflags/gflags.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
|
|
|
|
else
|
|
|
|
# check if namespace is google
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
|
|
|
|
#include <gflags/gflags.h>
|
|
|
|
using namespace google;
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_ZLIB; then
|
|
|
|
# Test whether zlib library is installed
|
|
|
|
$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <zlib.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_BZIP; then
|
|
|
|
# Test whether bzip library is installed
|
|
|
|
$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <bzlib.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_LZ4; then
|
|
|
|
# Test whether lz4 library is installed
|
|
|
|
$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <lz4.h>
|
|
|
|
#include <lz4hc.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_ZSTD; then
|
|
|
|
# Test whether zstd library is installed
|
|
|
|
$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <zstd.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_NUMA; then
|
|
|
|
# Test whether numa is available
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null <<EOF
|
|
|
|
#include <numa.h>
|
|
|
|
#include <numaif.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_TBB; then
|
|
|
|
# Test whether tbb is available
|
|
|
|
$CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null <<EOF
|
|
|
|
#include <tbb/tbb.h>
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DTBB"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_JEMALLOC; then
|
|
|
|
# Test whether jemalloc is available
|
|
|
|
if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
|
|
|
|
2>/dev/null; then
|
|
|
|
# This will enable some preprocessor identifiers in the Makefile
|
|
|
|
JEMALLOC=1
|
|
|
|
# JEMALLOC can be enabled either using the flag (like here) or by
|
|
|
|
# providing direct link to the jemalloc library
|
|
|
|
WITH_JEMALLOC_FLAG=1
|
|
|
|
# check for JEMALLOC installed with HomeBrew
|
|
|
|
if [ "$PLATFORM" == "OS_MACOSX" ]; then
|
|
|
|
if hash brew 2>/dev/null && brew ls --versions jemalloc > /dev/null; then
|
|
|
|
JEMALLOC_VER=$(brew ls --versions jemalloc | tail -n 1 | cut -f 2 -d ' ')
|
|
|
|
JEMALLOC_INCLUDE="-I/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/include"
|
|
|
|
JEMALLOC_LIB="/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $JEMALLOC_LIB"
|
|
|
|
JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS $JEMALLOC_LIB"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then
|
|
|
|
# jemalloc is not available. Let's try tcmalloc
|
|
|
|
if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
|
|
|
|
-ltcmalloc 2>/dev/null; then
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
|
|
|
|
fi
|
|
|
|
fi
|
Use malloc_usable_size() for accounting block cache size
Summary:
Currently, when we insert something into block cache, we say that the block cache capacity decreased by the size of the block. However, size of the block might be less than the actual memory used by this object. For example, 4.5KB block will actually use 8KB of memory. So even if we configure block cache to 10GB, our actually memory usage of block cache will be 20GB!
This problem showed up a lot in testing and just recently also showed up in MongoRocks production where we were using 30GB more memory than expected.
This diff will fix the problem. Instead of counting the block size, we will count memory used by the block. That way, a block cache configured to be 10GB will actually use only 10GB of memory.
I'm using non-portable function and I couldn't find info on portability on Google. However, it seems to work on Linux, which will cover majority of our use-cases.
Test Plan:
1. fill up mongo instance with 80GB of data
2. restart mongo with block cache size configured to 10GB
3. do a table scan in mongo
4. memory usage before the diff: 12GB. memory usage after the diff: 10.5GB
Reviewers: sdong, MarkCallaghan, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D40635
10 years ago
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_MALLOC_USABLE_SIZE; then
|
|
|
|
# Test whether malloc_usable_size is available
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <malloc.h>
|
|
|
|
int main() {
|
|
|
|
size_t res = malloc_usable_size(0);
|
|
|
|
(void)res;
|
|
|
|
return 0;
|
|
|
|
}
|
Use malloc_usable_size() for accounting block cache size
Summary:
Currently, when we insert something into block cache, we say that the block cache capacity decreased by the size of the block. However, size of the block might be less than the actual memory used by this object. For example, 4.5KB block will actually use 8KB of memory. So even if we configure block cache to 10GB, our actually memory usage of block cache will be 20GB!
This problem showed up a lot in testing and just recently also showed up in MongoRocks production where we were using 30GB more memory than expected.
This diff will fix the problem. Instead of counting the block size, we will count memory used by the block. That way, a block cache configured to be 10GB will actually use only 10GB of memory.
I'm using non-portable function and I couldn't find info on portability on Google. However, it seems to work on Linux, which will cover majority of our use-cases.
Test Plan:
1. fill up mongo instance with 80GB of data
2. restart mongo with block cache size configured to 10GB
3. do a table scan in mongo
4. memory usage before the diff: 12GB. memory usage after the diff: 10.5GB
Reviewers: sdong, MarkCallaghan, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D40635
10 years ago
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
|
|
|
|
fi
|
Use malloc_usable_size() for accounting block cache size
Summary:
Currently, when we insert something into block cache, we say that the block cache capacity decreased by the size of the block. However, size of the block might be less than the actual memory used by this object. For example, 4.5KB block will actually use 8KB of memory. So even if we configure block cache to 10GB, our actually memory usage of block cache will be 20GB!
This problem showed up a lot in testing and just recently also showed up in MongoRocks production where we were using 30GB more memory than expected.
This diff will fix the problem. Instead of counting the block size, we will count memory used by the block. That way, a block cache configured to be 10GB will actually use only 10GB of memory.
I'm using non-portable function and I couldn't find info on portability on Google. However, it seems to work on Linux, which will cover majority of our use-cases.
Test Plan:
1. fill up mongo instance with 80GB of data
2. restart mongo with block cache size configured to 10GB
3. do a table scan in mongo
4. memory usage before the diff: 12GB. memory usage after the diff: 10.5GB
Reviewers: sdong, MarkCallaghan, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D40635
10 years ago
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
|
|
|
|
# Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <pthread.h>
|
|
|
|
int main() {
|
|
|
|
int x = PTHREAD_MUTEX_ADAPTIVE_NP;
|
|
|
|
(void)x;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_BACKTRACE; then
|
|
|
|
# Test whether backtrace is available
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <execinfo.h>
|
|
|
|
int main() {
|
|
|
|
void* frames[1];
|
|
|
|
backtrace_symbols(frames, backtrace(frames, 1));
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
|
|
|
|
else
|
|
|
|
# Test whether execinfo library is installed
|
|
|
|
$CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <execinfo.h>
|
|
|
|
int main() {
|
|
|
|
void* frames[1];
|
|
|
|
backtrace_symbols(frames, backtrace(frames, 1));
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_PG; then
|
|
|
|
# Test if -pg is supported
|
|
|
|
$CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
int main() {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
PROFILING_FLAGS=-pg
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_SYNC_FILE_RANGE; then
|
|
|
|
# Test whether sync_file_range is supported for compatibility with an old glibc
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <fcntl.h>
|
|
|
|
int main() {
|
|
|
|
int fd = open("/dev/null", 0);
|
|
|
|
sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_SCHED_GETCPU; then
|
|
|
|
# Test whether sched_getcpu is supported
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <sched.h>
|
|
|
|
int main() {
|
|
|
|
int cpuid = sched_getcpu();
|
|
|
|
(void)cpuid;
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_ALIGNED_NEW; then
|
|
|
|
# Test whether c++17 aligned-new is supported
|
|
|
|
$CXX $PLATFORM_CXXFLAGS -faligned-new -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
struct alignas(1024) t {int a;};
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -faligned-new -DHAVE_ALIGNED_NEW"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
# TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
|
|
|
|
# -Wshorten-64-to-32 breaks compilation on FreeBSD i386
|
|
|
|
if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
|
|
|
|
# Test whether -Wshorten-64-to-32 is available
|
|
|
|
$CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null <<EOF
|
|
|
|
int main() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -Wshorten-64-to-32"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
# shall we use HDFS?
|
|
|
|
|
|
|
|
if test "$USE_HDFS"; then
|
|
|
|
if test -z "$JAVA_HOME"; then
|
|
|
|
echo "JAVA_HOME has to be set for HDFS usage." >&2
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS -I$HADOOP_HOME/include"
|
|
|
|
HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64 -L$HADOOP_HOME/lib/native"
|
|
|
|
HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib"
|
|
|
|
HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm"
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS"
|
|
|
|
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS"
|
|
|
|
JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if test -z "$PORTABLE"; then
|
|
|
|
if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
|
|
|
|
# Tune for this POWER processor, treating '+' models as base models
|
|
|
|
POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
|
|
|
|
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
|
|
|
|
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^arm`"; then
|
|
|
|
# TODO: Handle this with approprite options.
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS"
|
|
|
|
elif [ "$TARGET_OS" == "IOS" ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS"
|
|
|
|
elif [ "$TARGET_OS" != "AIX" ] && [ "$TARGET_OS" != "SunOS" ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -march=native "
|
|
|
|
elif test "$USE_SSE"; then
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
|
|
|
|
fi
|
|
|
|
elif test "$USE_SSE"; then
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
|
|
|
|
fi
|
|
|
|
|
|
|
|
$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
#include <cstdint>
|
|
|
|
#include <nmmintrin.h>
|
|
|
|
int main() {
|
|
|
|
volatile uint32_t x = _mm_crc32_u32(0, 0);
|
|
|
|
(void)x;
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
|
|
|
|
elif test "$USE_SSE"; then
|
|
|
|
echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling" >&2
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#include <cstdint>
|
|
|
|
#include <wmmintrin.h>
|
|
|
|
int main() {
|
|
|
|
const auto a = _mm_set_epi64x(0, 0);
|
|
|
|
const auto b = _mm_set_epi64x(0, 0);
|
|
|
|
const auto c = _mm_clmulepi64_si128(a, b, 0x00);
|
|
|
|
auto d = _mm_cvtsi128_si64(c);
|
|
|
|
(void)d;
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DHAVE_PCLMUL"
|
|
|
|
elif test "$USE_SSE"; then
|
|
|
|
echo "warning: USE_SSE specified but compiler could not use PCLMUL intrinsics, disabling" >&2
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
exit 1
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
fi
|
|
|
|
|
|
|
|
# iOS doesn't support thread-local storage, but this check would erroneously
|
|
|
|
# succeed because the cross-compiler flags are added by the Makefile, not this
|
|
|
|
# script.
|
|
|
|
if [ "$PLATFORM" != IOS ]; then
|
|
|
|
$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
|
|
|
#if defined(_MSC_VER) && !defined(__thread)
|
|
|
|
#define __thread __declspec(thread)
|
|
|
|
#endif
|
|
|
|
int main() {
|
|
|
|
static __thread int tls;
|
|
|
|
(void)tls;
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then
|
|
|
|
$CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null <<EOF
|
|
|
|
void dummy_func() {}
|
|
|
|
EOF
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
$CXX $COMMON_FLAGS $PLATFORM_SHARED_LDFLAGS test_dl.o -o /dev/null 2>/dev/null
|
|
|
|
if [ "$?" = 0 ]; then
|
|
|
|
EXEC_LDFLAGS+="-ldl"
|
|
|
|
rm -f test_dl.o
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
|
|
|
|
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
|
|
|
|
|
|
|
|
VALGRIND_VER="$VALGRIND_VER"
|
|
|
|
|
|
|
|
ROCKSDB_MAJOR=`build_tools/version.sh major`
|
|
|
|
ROCKSDB_MINOR=`build_tools/version.sh minor`
|
|
|
|
ROCKSDB_PATCH=`build_tools/version.sh patch`
|
|
|
|
|
|
|
|
echo "CC=$CC" >> "$OUTPUT"
|
|
|
|
echo "CXX=$CXX" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
|
|
|
|
echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
|
|
|
|
echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
|
|
|
|
echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> "$OUTPUT"
|
|
|
|
echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> "$OUTPUT"
|
|
|
|
echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> "$OUTPUT"
|
|
|
|
echo "JEMALLOC_INCLUDE=$JEMALLOC_INCLUDE" >> "$OUTPUT"
|
|
|
|
echo "JEMALLOC_LIB=$JEMALLOC_LIB" >> "$OUTPUT"
|
|
|
|
echo "ROCKSDB_MAJOR=$ROCKSDB_MAJOR" >> "$OUTPUT"
|
|
|
|
echo "ROCKSDB_MINOR=$ROCKSDB_MINOR" >> "$OUTPUT"
|
|
|
|
echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
|
|
|
|
echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
|
|
|
|
echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
|
|
|
|
echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
|
|
|
|
echo "FIND=$FIND" >> "$OUTPUT"
|
|
|
|
echo "WATCH=$WATCH" >> "$OUTPUT"
|
|
|
|
# This will enable some related identifiers for the preprocessor
|
|
|
|
if test -n "$JEMALLOC"; then
|
|
|
|
echo "JEMALLOC=1" >> "$OUTPUT"
|
|
|
|
fi
|
|
|
|
# Indicates that jemalloc should be enabled using -ljemalloc flag
|
|
|
|
# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
|
|
|
|
# and JEMALLOC_INCLUDE
|
|
|
|
if test -n "$WITH_JEMALLOC_FLAG"; then
|
|
|
|
echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
|
|
|
|
fi
|
|
|
|
echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
|