Add optional clang compile mode

Summary:
clang is an alternate compiler based on llvm.  It produces
nicer error messages and finds some bugs that gcc doesn't, such as the
size_t change in this file (which caused some write return values to be
misinterpreted!)

Clang isn't the default; to try it, do "USE_CLANG=1 make" or "export
USE_CLANG=1" then make as normal

Test Plan: "make check" and "USE_CLANG=1 make check"

Reviewers: dhruba

Reviewed By: dhruba

Differential Revision: https://reviews.facebook.net/D7899

build_detect_platform

@@ -30,7 +30,11 @@ fi
 
 # Default to fbcode gcc on internal fb machines
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
-    source fbcode.gcc471.sh
+    if [ -z "$USE_CLANG" ]; then
+        source fbcode.gcc471.sh
+    else
+        source fbcode.clang31.sh
+    fi
 fi
 
 # Delete existing output, if it exists
@@ -53,8 +57,8 @@ fi
 COMMON_FLAGS="${CFLAGS}"
 CROSS_COMPILE=
 PLATFORM_CCFLAGS=
-PLATFORM_CXXFLAGS=
+PLATFORM_CXXFLAGS="${CXXFLAGS}"
-PLATFORM_LDFLAGS=
+PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS"
 PLATFORM_SHARED_EXT="so"
 PLATFORM_SHARED_LDFLAGS="${EXEC_LDFLAGS_SHARED} -shared -Wl,-soname -Wl,"
 PLATFORM_SHARED_CFLAGS="-fPIC"
@@ -71,8 +75,11 @@ case "$TARGET_OS" in
         ;;
     Linux)
         PLATFORM=OS_LINUX
-        COMMON_FLAGS="$COMMON_FLAGS -I/usr/include -fno-builtin-memcmp -pthread -DOS_LINUX -fPIC"
+        COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX -fPIC"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
+        if [ -z "$USE_CLANG" ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
+        fi
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
         PORT_FILE=port/port_posix.cc
         ;;
     SunOS)

db/skiplist_test.cc

@@ -14,7 +14,7 @@ namespace leveldb {
 
 typedef uint64_t Key;
 
-struct Comparator {
+struct TestComparator {
   int operator()(const Key& a, const Key& b) const {
     if (a < b) {
       return -1;
@@ -30,11 +30,11 @@ class SkipTest { };
 
 TEST(SkipTest, Empty) {
   Arena arena;
-  Comparator cmp;
+  TestComparator cmp;
-  SkipList<Key, Comparator> list(cmp, &arena);
+  SkipList<Key, TestComparator> list(cmp, &arena);
   ASSERT_TRUE(!list.Contains(10));
 
-  SkipList<Key, Comparator>::Iterator iter(&list);
+  SkipList<Key, TestComparator>::Iterator iter(&list);
   ASSERT_TRUE(!iter.Valid());
   iter.SeekToFirst();
   ASSERT_TRUE(!iter.Valid());
@@ -50,8 +50,8 @@ TEST(SkipTest, InsertAndLookup) {
   Random rnd(1000);
   std::set<Key> keys;
   Arena arena;
-  Comparator cmp;
+  TestComparator cmp;
-  SkipList<Key, Comparator> list(cmp, &arena);
+  SkipList<Key, TestComparator> list(cmp, &arena);
   for (int i = 0; i < N; i++) {
     Key key = rnd.Next() % R;
     if (keys.insert(key).second) {
@@ -69,7 +69,7 @@ TEST(SkipTest, InsertAndLookup) {
 
   // Simple iterator tests
   {
-    SkipList<Key, Comparator>::Iterator iter(&list);
+    SkipList<Key, TestComparator>::Iterator iter(&list);
     ASSERT_TRUE(!iter.Valid());
 
     iter.Seek(0);
@@ -87,7 +87,7 @@ TEST(SkipTest, InsertAndLookup) {
 
   // Forward iteration test
   for (int i = 0; i < R; i++) {
-    SkipList<Key, Comparator>::Iterator iter(&list);
+    SkipList<Key, TestComparator>::Iterator iter(&list);
     iter.Seek(i);
 
     // Compare against model iterator
@@ -107,7 +107,7 @@ TEST(SkipTest, InsertAndLookup) {
 
   // Backward iteration test
   {
-    SkipList<Key, Comparator>::Iterator iter(&list);
+    SkipList<Key, TestComparator>::Iterator iter(&list);
     iter.SeekToLast();
 
     // Compare against model iterator
@@ -208,10 +208,10 @@ class ConcurrentTest {
 
   // SkipList is not protected by mu_.  We just use a single writer
   // thread to modify it.
-  SkipList<Key, Comparator> list_;
+  SkipList<Key, TestComparator> list_;
 
  public:
-  ConcurrentTest() : list_(Comparator(), &arena_) { }
+  ConcurrentTest() : list_(TestComparator(), &arena_) { }
 
   // REQUIRES: External synchronization
   void WriteStep(Random* rnd) {
@@ -230,7 +230,7 @@ class ConcurrentTest {
     }
 
     Key pos = RandomTarget(rnd);
-    SkipList<Key, Comparator>::Iterator iter(&list_);
+    SkipList<Key, TestComparator>::Iterator iter(&list_);
     iter.Seek(pos);
     while (true) {
       Key current;

db/version_set.cc

@@ -28,20 +28,6 @@ static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
   return sum;
 }
 
-namespace {
-std::string IntSetToString(const std::set<uint64_t>& s) {
-  std::string result = "{";
-  for (std::set<uint64_t>::const_iterator it = s.begin();
-       it != s.end();
-       ++it) {
-    result += (result.size() > 1) ? "," : "";
-    result += NumberToString(*it);
-  }
-  result += "}";
-  return result;
-}
-}  // namespace
-
 Version::~Version() {
   assert(refs_ == 0);
 

fbcode.clang31.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile leveldb using
+# fbcode settings.  It uses the latest g++ compiler and also
+# uses jemalloc
+# This is compiled with gcc version 4.7.1 for zeus proxy
+
+TOOLCHAIN_REV=f365dbeae46a30414a2874a6f45e73e10f1caf7d
+TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
+TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
+TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a
+GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1
+
+# location of snappy headers and libraries
+SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
+SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
+
+# location of boost headers and libraries
+THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x"
+THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
+THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib"
+
+# location of libevent
+LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
+LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=" -msse -msse4.2 "
+
+CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang $CLANG_INCLUDES"
+CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
+AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
+RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
+
+CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin -nostdlib -nostdinc -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1 -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/x86_64-facebook-linux -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/backward -isystem $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include -isystem $TOOLCHAIN_LIB_BASE/clang/clang-3.1/c8f7279/lib/clang/3.1/include -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include/linux -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include -Wall -Wno-sign-compare -Wno-unused-variable -Winvalid-pch -Wno-deprecated -Woverloaded-virtual"
+CXXFLAGS="$CFLAGS -nostdinc++ -std=gnu++0x"
+
+CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
+
+EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
+EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"
+
+PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS"
+SNAPPY_LDFLAGS="$SNAPPY_LIBS"
+
+export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED SNAPPY_LDFLAGS

fbcode.gcc471.sh

@@ -48,6 +48,8 @@ EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/l
 EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-20100812_experimental/91ddd43/lib/libunwind.a"
 EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
 
+PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
+
 EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS"
 SNAPPY_LDFLAGS="$SNAPPY_LIBS"
 

table/table_test.cc

@@ -411,12 +411,14 @@ static bool ZlibCompressionSupported() {
                              &out);
 }
 
+#ifdef BZIP2
 static bool BZip2CompressionSupported() {
   std::string out;
   Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
   return port::BZip2_Compress(Options().compression_opts, in.data(), in.size(),
                               &out);
 }
+#endif
 
 enum TestType {
   TABLE_TEST,

util/env_posix.cc

@@ -393,7 +393,7 @@ class PosixWritableFile : public WritableFile {
       cursize_ += left;
     } else {
       while (left != 0) {
-        size_t done = write(fd_, src, left);
+        ssize_t done = write(fd_, src, left);
         if (done < 0) {
           return IOError(filename_, errno);
         }
@@ -424,7 +424,7 @@ class PosixWritableFile : public WritableFile {
     size_t left = cursize_;
     char* src = buf_;
     while (left != 0) {
-      size_t done = write(fd_, src, left);
+      ssize_t done = write(fd_, src, left);
       if (done < 0) {
         return IOError(filename_, errno);
       }