Clean up the Makefile and the build scripts

Summary: As Aaron suggested, there are quite some problems with our Makefile and scripts. So in this diff I did some cleanup for them and revise some part of the scripts/makefile to help people better understand some mysterious parts.

Test Plan:
Ran make in several modes;
Ran the updated scripts.

Reviewers: dhruba, emayanke, akushner

Differential Revision: https://reviews.facebook.net/D12285
main
Kai Liu 12 years ago
parent 85d83a150b
commit 457dcc605a
  1. 3
      .gitignore
  2. 42
      Makefile
  3. 4
      VALGRIND_LOGS/README
  4. 30
      build_detect_version
  5. 10
      build_tools/build_detect_platform
  6. 42
      build_tools/build_detect_version
  7. 0
      build_tools/build_java.sh
  8. 0
      build_tools/fbcode.clang31.sh
  9. 0
      build_tools/fbcode.gcc471.sh
  10. 4
      build_tools/regression_build_test.sh
  11. 2
      build_tools/valgrind_test.sh
  12. 10
      coverage/coverage_test.sh
  13. 2
      e

3
.gitignore vendored

@ -18,3 +18,6 @@ leveldb_shell
manifest_dump
sst_dump
util/build_version.cc
build_tools/VALGRIND_LOGS/
coverage/COVERAGE_REPORT
util/build_version.cc.tmp

@ -14,7 +14,7 @@ OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
#-----------------------------------------------
# detect what platform we're building on
$(shell ./build_detect_platform build_config.mk)
$(shell (cd build_tools/; ROCKSDB_ROOT=.. ./build_detect_platform ../build_config.mk))
# this file is generated by the previous line to set build flags and sources
include build_config.mk
@ -31,7 +31,7 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
TESTUTIL = ./util/testutil.o
TESTHARNESS = ./util/testharness.o $(TESTUTIL)
VALGRIND_ERROR = 2
VALGRIND_DIR = VALGRIND_LOGS
VALGRIND_DIR = build_tools/VALGRIND_LOGS
VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
@ -109,7 +109,8 @@ endif # PLATFORM_SHARED_EXT
all: $(SHARED) $(LIBRARY) $(PROGRAMS)
.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests release tags valgrind_check whitebox_crash_test
.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests \
release tags valgrind_check whitebox_crash_test
release:
$(MAKE) clean
@ -120,7 +121,7 @@ coverage:
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" $(MAKE) all check
(cd coverage; ./coverage_test.sh)
# Delete intermediate files
find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs rm
find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
@ -137,6 +138,7 @@ whitebox_crash_test: db_stress
python -u tools/db_crashtest2.py
valgrind_check: all $(PROGRAMS) $(TESTS)
mkdir -p $(VALGRIND_DIR)
echo TESTS THAT HAVE VALGRIND ERRORS > $(VALGRIND_DIR)/valgrind_failed_tests; \
echo TIMES in seconds TAKEN BY TESTS ON VALGRIND > $(VALGRIND_DIR)/valgrind_tests_times; \
for t in $(filter-out skiplist_test,$(TESTS)); do \
@ -152,13 +154,19 @@ valgrind_check: all $(PROGRAMS) $(TESTS)
clean:
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) build_config.mk
-rm -rf ios-x86/* ios-arm/*
-find . -name "*.[od]" | xargs rm
-find . -name "*.[od]" | xargs --no-run-if-empty rm
-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
tags:
ctags * -R
cscope -b `find . -name '*.cc'` `find . -name '*.h'`
# ---------------------------------------------------------------------------
# Unit tests and tools
# ---------------------------------------------------------------------------
$(LIBRARY): $(LIBOBJECTS)
rm -f $@
$(AR) -rs $@ $(LIBOBJECTS)
db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
@ -283,9 +291,9 @@ sst_dump: tools/sst_dump.o $(LIBOBJECTS)
ldb: tools/ldb.o $(LIBOBJECTS)
$(CXX) tools/ldb.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
tags:
ctags `find . -name '*.cc'` `find . -name '*.h'`
cscope -b `find . -name '*.cc'` `find . -name '*.h'`
# ---------------------------------------------------------------------------
# Platform-specific compilation
# ---------------------------------------------------------------------------
ifeq ($(PLATFORM), IOS)
# For iOS, create universal object files to be used on both the simulator and
@ -317,9 +325,20 @@ else
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
endif
# ---------------------------------------------------------------------------
# Source files dependencies detection
# ---------------------------------------------------------------------------
# Add proper dependency support so changing a .h file forces a .cc file to
# rebuild.
# The .d file indicates .cc file's dependencies on .h files. We generate such
# dependency by g++'s -MM option, whose output is a make dependency rule.
# The sed command makes sure the "target" file in the generated .d file has
# the correct path prefix.
%.d: %.cc
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $*.d $(COVERAGEFLAGS)
@sed -i -e 's|.*:|$*.o:|' < $*.d
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $@
@sed -i -e 's|.*:|$*.o:|' $@
DEPFILES = $(filter-out util/build_version.d,$(SOURCES:.cc=.d))
@ -328,4 +347,3 @@ depend: $(DEPFILES)
ifneq ($(MAKECMDGOALS),clean)
-include $(DEPFILES)
endif

@ -1,4 +0,0 @@
This directory stores the tests that failed valgrind and the times associated
with the failed runs.
"make valgrind_check" can be invoked to call valgrind on the rocksdb tests and
generate files in this directory

@ -1,30 +0,0 @@
#!/bin/sh
#
# Record the version of the source that we are compiling.
# We keep a record of the git revision in util/version.cc. This source file
# is then built as a regular source file as part of the compilation process.
# One can run "strings executable_filename | grep _build_" to find the version of
# the source that we used to build the executable file.
#
# create git version file
VFILE=$(mktemp)
trap "rm $VFILE" EXIT
# check to see if git is in the path
which git > /dev/null
if [ "$?" = 0 ]; then
env -i git rev-parse HEAD | awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";"} END {}' > ${VFILE}
else
echo "git not found"| awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";"} END {}' > ${VFILE}
fi
date | awk 'BEGIN {} {print "const char * leveldb_build_git_datetime = \"leveldb_build_git_datetime:"$0"\";"} END {} ' >> ${VFILE}
echo "const char * leveldb_build_compile_date = __DATE__;" >> ${VFILE}
echo "const char * leveldb_build_compile_time = __TIME__;" >> ${VFILE}
OUTFILE=util/build_version.cc
if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
cp $VFILE $OUTFILE
fi

@ -31,9 +31,9 @@ fi
# Default to fbcode gcc on internal fb machines
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
if [ -z "$USE_CLANG" ]; then
source ./fbcode.gcc471.sh
source fbcode.gcc471.sh
else
source ./fbcode.clang31.sh
source fbcode.clang31.sh
fi
fi
@ -65,7 +65,7 @@ PLATFORM_SHARED_CFLAGS="-fPIC"
PLATFORM_SHARED_VERSIONED=true
# generic port files (working on all platform by #ifdef) go directly in /port
GENERIC_PORT_FILES=`find port -name '*.cc' | tr "\n" " "`
GENERIC_PORT_FILES=`cd $ROCKSDB_ROOT; find port -name '*.cc' | tr "\n" " "`
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
case "$TARGET_OS" in
@ -146,8 +146,8 @@ fi
set -f # temporarily disable globbing so that our patterns arent expanded
PRUNE_TEST="-name *test*.cc -prune"
PRUNE_BENCH="-name *_bench.cc -prune"
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
PORTABLE_CPP=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
PORTABLE_FILES=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
PORTABLE_CPP=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
set +f # re-enable globbing
# The sources consist of the portable files, plus the platform-specific port

@ -0,0 +1,42 @@
#!/bin/sh
#
# Record the version of the source that we are compiling.
# We keep a record of the git revision in util/version.cc. This source file
# is then built as a regular source file as part of the compilation process.
# One can run "strings executable_filename | grep _build_" to find the version of
# the source that we used to build the executable file.
#
# create git version file
VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp
trap "rm $VFILE" EXIT
# check to see if git is in the path
which git > /dev/null
if [ "$?" = 0 ]; then
env -i git rev-parse HEAD |
awk '
BEGIN {
print "#include \"build_version.h\"\n"
}
{ print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";" }
' > ${VFILE}
else
echo "git not found" |
awk '
BEGIN {
print "#include \"build_version.h\""
}
{ print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";" }
' > ${VFILE}
fi
echo "const char* leveldb_build_git_datetime = \"leveldb_build_git_datetime:$(date)\";" >> ${VFILE}
echo "const char* leveldb_build_compile_date = __DATE__;" >> ${VFILE}
echo "const char* leveldb_build_compile_time = __TIME__;" >> ${VFILE}
OUTFILE=$ROCKSDB_ROOT/util/build_version.cc
if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
cp $VFILE $OUTFILE
fi

@ -14,8 +14,8 @@ fi
# On the production build servers, set data and stat
# files/directories not in /tmp or else the tempdir cleaning
# scripts will make you very unhappy.
DATA_DIR=${DATA_DIR:-$(mktemp --tmpdir -d rocksdb_XXXX)}
STAT_FILE=${STAT_FILE:-$(mktemp --tmpdir -u rocksdb_test_stats_XXXX)}
DATA_DIR=${DATA_DIR:-$(mktemp -t -d rocksdb_XXXX)}
STAT_FILE=${STAT_FILE:-$(mktemp -t -u rocksdb_test_stats_XXXX)}
function cleanup {
rm -rf $DATA_DIR

@ -2,7 +2,7 @@
#A shell script for Jenknis to run valgrind on rocksdb tests
#Returns 0 on success when there are no failed tests
VALGRIND_DIR=VALGRIND_LOGS
VALGRIND_DIR=build_tools/VALGRIND_LOGS
make clean
make -j$(nproc) valgrind_check
NUM_FAILED_TESTS=$((`wc -l $VALGRIND_DIR/valgrind_failed_tests | awk '{print $1}'` - 1))

@ -11,13 +11,13 @@ fi
ROOT=".."
# Fetch right version of gcov
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
source $ROOT/fbcode.gcc471.sh
source $ROOT/build_tools/fbcode.gcc471.sh
GCOV=$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1/cc6c9dc/bin/gcov
else
GCOV=$(which gcov)
fi
COVERAGE_DIR=$(mktemp -t -d rocksdb_coverage_XXXX)
COVERAGE_DIR="$PWD/COVERAGE_REPORT"
mkdir -p $COVERAGE_DIR
# Find all gcno files to generate the coverage report
@ -47,16 +47,14 @@ echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
# Generate the html report. If we cannot find lcov in this machine, we'll simply
# skip this step.
echo "Generating the html coverage report..."
set +e
LCOV=$(which lcov 2>/dev/null)
LCOV=$(which lcov || true 2>/dev/null)
if [ -z $LCOV ]
then
echo "Skip: Cannot find lcov to generate the html report."
exit 0
fi
set -e
(cd $ROOT; lcov --no-external \
--capture \
--directory $PWD \

2
e

@ -1,2 +0,0 @@
ar: creating libmemenv.a
ar: creating libleveldb.a
Loading…
Cancel
Save