Dbid feature

Summary:
Create a new type of file on startup if it doesn't already exist called DBID.
This will store a unique number generated from boost library's uuid header file.
The use-case is to identify the case of a db losing all its data and coming back up either empty or from an image(backup/live replica's recovery)
the key point to note is that DBID is not stored in a backup or db snapshot
It's preferable to use Boost for uuid because:
1) A non-standard way of generating uuid is not good
2) /proc/sys/kernel/random/uuid generates a uuid but only on linux environments and the solution would not be clean
3) c++ doesn't have any direct way to get a uuid
4) Boost is a very good library that was already having linkage in rocksdb from third-party
Note: I had to update the TOOLCHAIN_REV in build files to get latest verison of boost from third-party as the older version had a bug.
I had to put Wno-uninitialized in Makefile because boost-1.51 has an unitialized variable and rocksdb would not comiple otherwise. Latet open-source for boost is 1.54 but is not there in third-party. I have notified the concerned people in fbcode about it.
@kailiu : While releasing to third-party, an additional dependency will need to be created for boost in TARGETS file. I can help identify.

Test Plan:
Expand db_test to test 2 cases
1) Restarting db with Id file present - verify that no change to Id
2)Restarting db with Id file deleted - verify that a different Id is there after reopen
Also run make all check

Reviewers: dhruba, haobo, kailiu, sdong

Reviewed By: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D13587

build_tools/build_detect_platform

@@ -134,14 +134,6 @@ $PWD/build_tools/build_detect_version
 # of all files matching either rule, so we need to append -print to make the
 # prune take effect.
 DIRS="util db table utilities"
-if test "$USE_THRIFT"; then
-  DIRS="$DIRS thrift/server_utils.cpp thrift/gen-cpp "
-  THRIFTSERVER=leveldb_server
-fi
-
-if test "$USE_SCRIBE"; then
-  DIRS="$DIRS scribe "
-fi
 
 set -f # temporarily disable globbing so that our patterns arent expanded
 PRUNE_TEST="-name *test*.cc -prune"
@@ -227,14 +219,6 @@ if test "$USE_HDFS"; then
   PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS"
 fi
 
-# shall we build thrift server or scribe logger
-if test "$USE_THRIFT" ||  test "$USE_SCRIBE" ; then
-    THRIFT_CCFLAGS=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp -I/usr/include -std=gnu++0x"
-    THRIFT_LDFLAGS=" -lexample -lserver -lthrift_base -ltransport -lthrift_exception -lutil -L./thrift/libs "
-    COMMON_FLAGS="$COMMON_FLAGS $THRIFT_CCFLAGS"
-    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $THRIFT_LDFLAGS"
-fi
-
 # if Intel SSE instruction set is supported, set USE_SSE=" -msse -msse4.2 "
 COMMON_FLAGS="$COMMON_FLAGS $USE_SSE"
 
@@ -254,5 +238,4 @@ echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
 echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
 echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
 echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
-echo "THRIFTSERVER=$THRIFTSERVER" >> $OUTPUT
 echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> $OUTPUT

build_tools/fbcode.clang31.sh

@@ -4,21 +4,16 @@
 # fbcode settings.  It uses the latest g++ compiler and also
 # uses jemalloc
 
-TOOLCHAIN_REV=f365dbeae46a30414a2874a6f45e73e10f1caf7d
+TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
 TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
 TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
-TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a
+TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
 GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1
 
 # location of snappy headers and libraries
 SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
 SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
 
-# location of boost headers and libraries
-THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x"
-THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
-THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib"
-
 # location of libevent
 LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
 LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
@@ -27,7 +22,7 @@ LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
 export USE_SSE=" -msse -msse4.2 "
 
 CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang $CLANG_INCLUDES"
-CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
+CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $LIBEVENT_INCLUDE"
 AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
 RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
 
@@ -37,8 +32,8 @@ CXXFLAGS="$CFLAGS -nostdinc++ -std=gnu++0x"
 CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
 
 EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
-EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
-EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $LIBEVENT_LIBS"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
 EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"
 

build_tools/fbcode.gcc471.sh

@@ -4,10 +4,10 @@
 # fbcode settings.  It uses the latest g++ compiler and also
 # uses jemalloc
 
-TOOLCHAIN_REV=83eb773e262fa705eaebbf3de40db71d53fabf2e
+TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
 TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
 TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
-TOOL_JEMALLOC=jemalloc-3.3.1/2f45f3a
+TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
 
 # location of libhdfs libraries
 if test "$USE_HDFS"; then
@@ -27,11 +27,6 @@ SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
 ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
 ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
 
-# location of boost headers and libraries
-THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/include"
-THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
-THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/lib"
-
 # location of libevent
 LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
 LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
@@ -40,7 +35,7 @@ LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
 export USE_SSE=" -msse -msse4.2 "
 
 CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc"
-CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
+CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $LIBEVENT_INCLUDE"
 AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
 RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
 
@@ -48,8 +43,8 @@ CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/bin/gold -m64 -mtune=g
 CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
 
 EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
-EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
-EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $LIBEVENT_LIBS"
 
 PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
 

db/db_impl.cc

@@ -509,6 +509,7 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
           break;
         case kCurrentFile:
         case kDBLockFile:
+        case kIdentityFile:
         case kMetaDatabase:
           keep = true;
           break;
@@ -639,6 +640,13 @@ Status DBImpl::Recover(VersionEdit* edit, MemTable* external_table,
             dbname_, "exists (error_if_exists is true)");
       }
     }
+    // Check for the IDENTITY file and create it if not there
+    if (!env_->FileExists(IdentityFileName(dbname_))) {
+      s = SetIdentityFile(env_, dbname_);
+      if (!s.ok()) {
+        return s;
+      }
+    }
   }
 
   Status s = versions_->Recover();

db/db_test.cc

@@ -1630,12 +1630,12 @@ TEST(DBTest, ManifestRollOver) {
       ASSERT_OK(Put("manifest_key1", std::string(1000, '1')));
       ASSERT_OK(Put("manifest_key2", std::string(1000, '2')));
       ASSERT_OK(Put("manifest_key3", std::string(1000, '3')));
-      uint64_t manifest_before_fulsh =
+      uint64_t manifest_before_flush =
         dbfull()->TEST_Current_Manifest_FileNo();
       dbfull()->Flush(FlushOptions()); // This should trigger LogAndApply.
       uint64_t manifest_after_flush =
         dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_GT(manifest_after_flush, manifest_before_fulsh);
+      ASSERT_GT(manifest_after_flush, manifest_before_flush);
       Reopen(&options);
       ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(),
                 manifest_after_flush);
@@ -1647,6 +1647,36 @@ TEST(DBTest, ManifestRollOver) {
   } while (ChangeCompactOptions());
 }
 
+TEST(DBTest, IdentityAcrossRestarts) {
+  do {
+    std::string idfilename = IdentityFileName(dbname_);
+    unique_ptr<SequentialFile> idfile;
+    const EnvOptions soptions;
+    ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions));
+    char buffer1[100];
+    Slice id1;
+    ASSERT_OK(idfile->Read(100, &id1, buffer1));
+
+    Options options = CurrentOptions();
+    Reopen(&options);
+    char buffer2[100];
+    Slice id2;
+    ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions));
+    ASSERT_OK(idfile->Read(100, &id2, buffer2));
+    // id1 should match id2 because identity was not regenerated
+    ASSERT_EQ(id1.ToString(), id2.ToString());
+
+    ASSERT_OK(env_->DeleteFile(idfilename));
+    Reopen(&options);
+    char buffer3[100];
+    Slice id3;
+    ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions));
+    ASSERT_OK(idfile->Read(100, &id3, buffer3));
+    // id1 should NOT match id2 because identity was regenerated
+    ASSERT_NE(id1.ToString(0), id3.ToString());
+  } while (ChangeCompactOptions());
+}
+
 TEST(DBTest, RecoverWithLargeLog) {
   do {
     {

db/deletefile_test.cc

@@ -14,12 +14,11 @@
 #include "db/write_batch_internal.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
-#include "boost/lexical_cast.hpp"
 #include "rocksdb/env.h"
 #include <vector>
-#include <boost/algorithm/string.hpp>
 #include <stdlib.h>
 #include <map>
+#include <string>
 
 namespace rocksdb {
 
@@ -62,7 +61,7 @@ class DeleteFileTest {
     options.sync = false;
     ReadOptions roptions;
     for (int i = startkey; i < (numkeys + startkey) ; i++) {
-      std::string temp = boost::lexical_cast<std::string>(i);
+      std::string temp = std::to_string(i);
       Slice key(temp);
       Slice value(temp);
       ASSERT_OK(db_->Put(options, key, value));

db/filename.cc

@@ -127,7 +127,12 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
   return dbname + buf;
 }
 
+std::string IdentityFileName(const std::string& dbname) {
+  return dbname + "/IDENTITY";
+}
+
 // Owned filenames have the form:
+//    dbname/IDENTITY
 //    dbname/CURRENT
 //    dbname/LOCK
 //    dbname/LOG
@@ -143,7 +148,10 @@ bool ParseFileName(const std::string& fname,
   if (fname.length() > 1 && fname[0] == '/') {
     rest.remove_prefix(1);
   }
-  if (rest == "CURRENT") {
+  if (rest == "IDENTITY") {
+    *number = 0;
+    *type = kIdentityFile;
+  } else if (rest == "CURRENT") {
     *number = 0;
     *type = kCurrentFile;
   } else if (rest == "LOCK") {
@@ -223,4 +231,18 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
   return s;
 }
 
+Status SetIdentityFile(Env* env, const std::string& dbname) {
+  std::string id = env->GenerateUniqueId();
+  assert(!id.empty());
+  std::string tmp = TempFileName(dbname, id.size());
+  Status s = WriteStringToFileSync(env, id, tmp);
+  if (s.ok()) {
+    s = env->RenameFile(tmp, IdentityFileName(dbname));
+  }
+  if (!s.ok()) {
+    env->DeleteFile(tmp);
+  }
+  return s;
+}
+
 }  // namespace rocksdb

db/filename.h

@@ -28,7 +28,8 @@ enum FileType {
   kCurrentFile,
   kTempFile,
   kInfoLogFile,  // Either the current one, or an old one
-  kMetaDatabase
+  kMetaDatabase,
+  kIdentityFile
 };
 
 // Return the name of the log file with the specified number
@@ -82,6 +83,11 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
 extern std::string MetaDatabaseName(const std::string& dbname,
                                     uint64_t number);
 
+// Return the name of the Identity file which stores a unique number for the db
+// that will get regenerated if the db loses all its data and is recreated fresh
+// either from a backup-image or empty
+extern std::string IdentityFileName(const std::string& dbname);
+
 // If filename is a rocksdb file, store the type of the file in *type.
 // The number encoded in the filename is stored in *number.  If the
 // filename was successfully parsed, returns true.  Else return false.
@@ -94,5 +100,7 @@ extern bool ParseFileName(const std::string& filename,
 extern Status SetCurrentFile(Env* env, const std::string& dbname,
                              uint64_t descriptor_number);
 
+// Make the IDENTITY file for the db
+extern Status SetIdentityFile(Env* env, const std::string& dbname);
 
 }  // namespace rocksdb

include/rocksdb/env.h

@@ -233,6 +233,9 @@ class Env {
   // Converts seconds-since-Jan-01-1970 to a printable string
   virtual std::string TimeToString(uint64_t time) = 0;
 
+  // Generates a unique id that can be used to identify a db
+  virtual std::string GenerateUniqueId();
+
  private:
   // No copying allowed
   Env(const Env&);

util/env_posix.cc

@@ -1386,6 +1386,25 @@ void PosixEnv::StartThread(void (*function)(void* arg), void* arg) {
 
 }  // namespace
 
+std::string Env::GenerateUniqueId() {
+  std::string uuid_file = "/proc/sys/kernel/random/uuid";
+  if (FileExists(uuid_file)) {
+    std::string uuid;
+    Status s = ReadFileToString(this, uuid_file, &uuid);
+    if (s.ok()) {
+      return uuid;
+    }
+  }
+  // Could not read uuid_file - generate uuid using "nanos-random"
+  Random64 r(time(nullptr));
+  uint64_t random_uuid_portion =
+    r.Uniform(std::numeric_limits<uint64_t>::max());
+  uint64_t nanos_uuid_portion = NowNanos();
+  char uuid2[200];
+  snprintf(uuid2, 200, "%lx-%lx", nanos_uuid_portion, random_uuid_portion);
+  return uuid2;
+}
+
 Env* Env::Default() {
   static PosixEnv default_env;
   return &default_env;