From 994b3bd6933649d6c358feb819600298c3313128 Mon Sep 17 00:00:00 2001 From: Gunnar Kudrjavets Date: Wed, 30 Mar 2016 15:59:24 -0700 Subject: [PATCH] Add support for UBsan builds to RocksDB Summary: Undefined Behavior Sanitizer (ubsan) is //a good thing// which will help us to find sneaky bugs with low cost. Please see http://developerblog.redhat.com/2014/10/16/gcc-undefined-behavior-sanitizer-ubsan/ for more details and official GCC documentation for more context: https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html. Changes itself are quite simple and pretty much imitating whatever is implemented for ASan. Hooking the UBsan validation build to Sandcastle is a separate step and will be dealt as separate diff because code is in internal repository. Test Plan: Make sure that that there no regressions when it comes to builds and test pass rate. Reviewers: leveldb, sdong, IslamAbdelRahman, yhchiang Reviewed By: yhchiang Subscribers: andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D56049 --- Makefile | 23 +++++++++++- build_tools/precommit_checker.py | 2 +- build_tools/rocksdb-lego-determinator | 53 ++++++++++++++++++++++++++- tools/sst_dump_tool.cc | 5 ++- 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b5eb097fc..9396798b4 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,15 @@ else pg = -pg endif +# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc. +ifdef COMPILE_WITH_UBSAN + DISABLE_JEMALLOC=1 + EXEC_LDFLAGS += -fsanitize=undefined + PLATFORM_CCFLAGS += -fsanitize=undefined + PLATFORM_CXXFLAGS += -fsanitize=undefined +endif + + ifndef DISABLE_JEMALLOC EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS) PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE) @@ -600,7 +609,7 @@ ldb_tests: ldb crash_test: whitebox_crash_test blackbox_crash_test blackbox_crash_test: db_stress - python -u tools/db_crashtest.py --simple blackbox + python -u tools/db_crashtest.py --simple blackbox python -u tools/db_crashtest.py blackbox whitebox_crash_test: db_stress @@ -617,6 +626,16 @@ asan_crash_test: COMPILE_WITH_ASAN=1 $(MAKE) crash_test $(MAKE) clean +ubsan_check: + $(MAKE) clean + COMPILE_WITH_UBSAN=1 $(MAKE) check -j32 + $(MAKE) clean + +ubsan_crash_test: + $(MAKE) clean + COMPILE_WITH_UBSAN=1 $(MAKE) crash_test + $(MAKE) clean + valgrind_check: $(TESTS) for t in $(filter-out %skiplist_test,$(TESTS)); do \ $(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \ @@ -1221,7 +1240,7 @@ jdb_bench: commit_prereq: build_tools/rocksdb-lego-determinator \ build_tools/precommit_checker.py - J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan lite + J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan ubsan lite $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava; xfunc: diff --git a/build_tools/precommit_checker.py b/build_tools/precommit_checker.py index ceb5cb4ab..e47554772 100755 --- a/build_tools/precommit_checker.py +++ b/build_tools/precommit_checker.py @@ -184,7 +184,7 @@ parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.') # parser.add_argument('test', nargs='+', - help='CI test(s) to run. e.g: unit punit asan tsan') + help='CI test(s) to run. e.g: unit punit asan tsan ubsan') print("Please follow log %s" % Log.LOG_FILE) diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator index 1cad195da..cac74db15 100755 --- a/build_tools/rocksdb-lego-determinator +++ b/build_tools/rocksdb-lego-determinator @@ -67,6 +67,7 @@ ASAN="COMPILE_WITH_ASAN=1" CLANG="USE_CLANG=1" LITE="OPT=-DROCKSDB_LITE" TSAN="COMPILE_WITH_TSAN=1" +UBSAN="COMPILE_WITH_UBSAN=1" DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" PARSER="'parser':'egrep \'Failure|^#|Abort|Expected|Actual|GoogleTestFailure|^==\''" @@ -376,6 +377,48 @@ ASAN_CRASH_TEST_COMMANDS="[ } ]" +# +# RocksDB test under undefined behavior sanitizer +# +UBSAN_TEST_COMMANDS="[ + { + 'name':'Rocksdb Unit Test under UBSAN', + 'oncall':'$ONCALL', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Test RocksDB debug under UBSAN', + 'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check', + 'user':'root', + $PARSER + } + ], + $REPORT + } +]" + +# +# RocksDB crash testing under udnefined behavior sanitizer +# +UBSAN_CRASH_TEST_COMMANDS="[ + { + 'name':'Rocksdb crash test under UBSAN', + 'oncall':'$ONCALL', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug ubsan_crash_test', + 'timeout': 86400, + 'shell':'$SHM $DEBUG make J=1 ubsan_crash_test', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + # # RocksDB unit test under valgrind # @@ -462,9 +505,9 @@ run_format_compatible() if [ -e "build_detect_platform" ] then sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file - rm -rf build_detect_platform + rm -rf build_detect_platform cp $TEST_TMPDIR/temp_build_file build_detect_platform - chmod +x build_detect_platform + chmod +x build_detect_platform fi make ldb -j32 @@ -632,6 +675,12 @@ case $1 in asan_crash) echo $ASAN_CRASH_TEST_COMMANDS ;; + ubsan) + echo $UBSAN_TEST_COMMANDS + ;; + ubsan_crash) + echo $UBSAN_CRASH_TEST_COMMANDS + ;; valgrind) echo $VALGRIND_TEST_COMMANDS ;; diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index 316ee4045..5724b42c1 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -59,7 +59,10 @@ extern const uint64_t kLegacyPlainTableMagicNumber; const char* testFileName = "test_file_name"; Status SstFileReader::GetTableReader(const std::string& file_path) { - uint64_t magic_number; + // Warning about 'magic_number' being uninitialized shows up only in UBsan + // builds. Though access is guarded by 's.ok()' checks, fix the issue to + // avoid any warnings. + uint64_t magic_number = Footer::kInvalidTableMagicNumber; // read table magic number Footer footer;