From 9b50106f9ae1163c2da55b479bc01035964ac392 Mon Sep 17 00:00:00 2001 From: Mayank Agarwal Date: Fri, 18 Oct 2013 14:50:54 -0700 Subject: [PATCH] Dbid feature Summary: Create a new type of file on startup if it doesn't already exist called DBID. This will store a unique number generated from boost library's uuid header file. The use-case is to identify the case of a db losing all its data and coming back up either empty or from an image(backup/live replica's recovery) the key point to note is that DBID is not stored in a backup or db snapshot It's preferable to use Boost for uuid because: 1) A non-standard way of generating uuid is not good 2) /proc/sys/kernel/random/uuid generates a uuid but only on linux environments and the solution would not be clean 3) c++ doesn't have any direct way to get a uuid 4) Boost is a very good library that was already having linkage in rocksdb from third-party Note: I had to update the TOOLCHAIN_REV in build files to get latest verison of boost from third-party as the older version had a bug. I had to put Wno-uninitialized in Makefile because boost-1.51 has an unitialized variable and rocksdb would not comiple otherwise. Latet open-source for boost is 1.54 but is not there in third-party. I have notified the concerned people in fbcode about it. @kailiu : While releasing to third-party, an additional dependency will need to be created for boost in TARGETS file. I can help identify. Test Plan: Expand db_test to test 2 cases 1) Restarting db with Id file present - verify that no change to Id 2)Restarting db with Id file deleted - verify that a different Id is there after reopen Also run make all check Reviewers: dhruba, haobo, kailiu, sdong Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D13587 --- build_tools/build_detect_platform | 17 ---------------- build_tools/fbcode.clang31.sh | 15 +++++--------- build_tools/fbcode.gcc471.sh | 15 +++++--------- db/db_impl.cc | 8 ++++++++ db/db_test.cc | 34 +++++++++++++++++++++++++++++-- db/deletefile_test.cc | 5 ++--- db/filename.cc | 24 +++++++++++++++++++++- db/filename.h | 10 ++++++++- include/rocksdb/env.h | 3 +++ util/env_posix.cc | 19 +++++++++++++++++ 10 files changed, 106 insertions(+), 44 deletions(-) diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 884f2e836..c326b055c 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -134,14 +134,6 @@ $PWD/build_tools/build_detect_version # of all files matching either rule, so we need to append -print to make the # prune take effect. DIRS="util db table utilities" -if test "$USE_THRIFT"; then - DIRS="$DIRS thrift/server_utils.cpp thrift/gen-cpp " - THRIFTSERVER=leveldb_server -fi - -if test "$USE_SCRIBE"; then - DIRS="$DIRS scribe " -fi set -f # temporarily disable globbing so that our patterns arent expanded PRUNE_TEST="-name *test*.cc -prune" @@ -227,14 +219,6 @@ if test "$USE_HDFS"; then PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS" fi -# shall we build thrift server or scribe logger -if test "$USE_THRIFT" || test "$USE_SCRIBE" ; then - THRIFT_CCFLAGS=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp -I/usr/include -std=gnu++0x" - THRIFT_LDFLAGS=" -lexample -lserver -lthrift_base -ltransport -lthrift_exception -lutil -L./thrift/libs " - COMMON_FLAGS="$COMMON_FLAGS $THRIFT_CCFLAGS" - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $THRIFT_LDFLAGS" -fi - # if Intel SSE instruction set is supported, set USE_SSE=" -msse -msse4.2 " COMMON_FLAGS="$COMMON_FLAGS $USE_SSE" @@ -254,5 +238,4 @@ echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT -echo "THRIFTSERVER=$THRIFTSERVER" >> $OUTPUT echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> $OUTPUT diff --git a/build_tools/fbcode.clang31.sh b/build_tools/fbcode.clang31.sh index bcedad200..408f77022 100644 --- a/build_tools/fbcode.clang31.sh +++ b/build_tools/fbcode.clang31.sh @@ -4,21 +4,16 @@ # fbcode settings. It uses the latest g++ compiler and also # uses jemalloc -TOOLCHAIN_REV=f365dbeae46a30414a2874a6f45e73e10f1caf7d +TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native" TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1" -TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a +TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3 GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1 # location of snappy headers and libraries SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include" SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a" -# location of boost headers and libraries -THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x" -THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp" -THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib" - # location of libevent LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include" LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib" @@ -27,7 +22,7 @@ LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib" export USE_SSE=" -msse -msse4.2 " CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang $CLANG_INCLUDES" -CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE" +CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $LIBEVENT_INCLUDE" AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib @@ -37,8 +32,8 @@ CXXFLAGS="$CFLAGS -nostdinc++ -std=gnu++0x" CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC" EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a" -EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a" -EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS" +EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a" +EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $LIBEVENT_LIBS" EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2" EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin" diff --git a/build_tools/fbcode.gcc471.sh b/build_tools/fbcode.gcc471.sh index b80393f70..bd24f4c7d 100644 --- a/build_tools/fbcode.gcc471.sh +++ b/build_tools/fbcode.gcc471.sh @@ -4,10 +4,10 @@ # fbcode settings. It uses the latest g++ compiler and also # uses jemalloc -TOOLCHAIN_REV=83eb773e262fa705eaebbf3de40db71d53fabf2e +TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native" TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1" -TOOL_JEMALLOC=jemalloc-3.3.1/2f45f3a +TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3 # location of libhdfs libraries if test "$USE_HDFS"; then @@ -27,11 +27,6 @@ SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a" ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include" ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a" -# location of boost headers and libraries -THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/include" -THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp" -THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/lib" - # location of libevent LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include" LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib" @@ -40,7 +35,7 @@ LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib" export USE_SSE=" -msse -msse4.2 " CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc" -CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE" +CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $LIBEVENT_INCLUDE" AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib @@ -48,8 +43,8 @@ CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/bin/gold -m64 -mtune=g CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC" EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a" -EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a" -EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $THRIFT_LIBS $LIBEVENT_LIBS" +EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a" +EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $LIBEVENT_LIBS" PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib" diff --git a/db/db_impl.cc b/db/db_impl.cc index 812c679b7..d7c10261c 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -509,6 +509,7 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) { break; case kCurrentFile: case kDBLockFile: + case kIdentityFile: case kMetaDatabase: keep = true; break; @@ -639,6 +640,13 @@ Status DBImpl::Recover(VersionEdit* edit, MemTable* external_table, dbname_, "exists (error_if_exists is true)"); } } + // Check for the IDENTITY file and create it if not there + if (!env_->FileExists(IdentityFileName(dbname_))) { + s = SetIdentityFile(env_, dbname_); + if (!s.ok()) { + return s; + } + } } Status s = versions_->Recover(); diff --git a/db/db_test.cc b/db/db_test.cc index 074a902e9..53a5347c8 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -1630,12 +1630,12 @@ TEST(DBTest, ManifestRollOver) { ASSERT_OK(Put("manifest_key1", std::string(1000, '1'))); ASSERT_OK(Put("manifest_key2", std::string(1000, '2'))); ASSERT_OK(Put("manifest_key3", std::string(1000, '3'))); - uint64_t manifest_before_fulsh = + uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo(); dbfull()->Flush(FlushOptions()); // This should trigger LogAndApply. uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo(); - ASSERT_GT(manifest_after_flush, manifest_before_fulsh); + ASSERT_GT(manifest_after_flush, manifest_before_flush); Reopen(&options); ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush); @@ -1647,6 +1647,36 @@ TEST(DBTest, ManifestRollOver) { } while (ChangeCompactOptions()); } +TEST(DBTest, IdentityAcrossRestarts) { + do { + std::string idfilename = IdentityFileName(dbname_); + unique_ptr idfile; + const EnvOptions soptions; + ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions)); + char buffer1[100]; + Slice id1; + ASSERT_OK(idfile->Read(100, &id1, buffer1)); + + Options options = CurrentOptions(); + Reopen(&options); + char buffer2[100]; + Slice id2; + ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions)); + ASSERT_OK(idfile->Read(100, &id2, buffer2)); + // id1 should match id2 because identity was not regenerated + ASSERT_EQ(id1.ToString(), id2.ToString()); + + ASSERT_OK(env_->DeleteFile(idfilename)); + Reopen(&options); + char buffer3[100]; + Slice id3; + ASSERT_OK(env_->NewSequentialFile(idfilename, &idfile, soptions)); + ASSERT_OK(idfile->Read(100, &id3, buffer3)); + // id1 should NOT match id2 because identity was regenerated + ASSERT_NE(id1.ToString(0), id3.ToString()); + } while (ChangeCompactOptions()); +} + TEST(DBTest, RecoverWithLargeLog) { do { { diff --git a/db/deletefile_test.cc b/db/deletefile_test.cc index 92f9e5559..406b6b129 100644 --- a/db/deletefile_test.cc +++ b/db/deletefile_test.cc @@ -14,12 +14,11 @@ #include "db/write_batch_internal.h" #include "util/testharness.h" #include "util/testutil.h" -#include "boost/lexical_cast.hpp" #include "rocksdb/env.h" #include -#include #include #include +#include namespace rocksdb { @@ -62,7 +61,7 @@ class DeleteFileTest { options.sync = false; ReadOptions roptions; for (int i = startkey; i < (numkeys + startkey) ; i++) { - std::string temp = boost::lexical_cast(i); + std::string temp = std::to_string(i); Slice key(temp); Slice value(temp); ASSERT_OK(db_->Put(options, key, value)); diff --git a/db/filename.cc b/db/filename.cc index c80226f07..786861026 100644 --- a/db/filename.cc +++ b/db/filename.cc @@ -127,7 +127,12 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) { return dbname + buf; } +std::string IdentityFileName(const std::string& dbname) { + return dbname + "/IDENTITY"; +} + // Owned filenames have the form: +// dbname/IDENTITY // dbname/CURRENT // dbname/LOCK // dbname/LOG @@ -143,7 +148,10 @@ bool ParseFileName(const std::string& fname, if (fname.length() > 1 && fname[0] == '/') { rest.remove_prefix(1); } - if (rest == "CURRENT") { + if (rest == "IDENTITY") { + *number = 0; + *type = kIdentityFile; + } else if (rest == "CURRENT") { *number = 0; *type = kCurrentFile; } else if (rest == "LOCK") { @@ -223,4 +231,18 @@ Status SetCurrentFile(Env* env, const std::string& dbname, return s; } +Status SetIdentityFile(Env* env, const std::string& dbname) { + std::string id = env->GenerateUniqueId(); + assert(!id.empty()); + std::string tmp = TempFileName(dbname, id.size()); + Status s = WriteStringToFileSync(env, id, tmp); + if (s.ok()) { + s = env->RenameFile(tmp, IdentityFileName(dbname)); + } + if (!s.ok()) { + env->DeleteFile(tmp); + } + return s; +} + } // namespace rocksdb diff --git a/db/filename.h b/db/filename.h index c5a4cde2c..463be4094 100644 --- a/db/filename.h +++ b/db/filename.h @@ -28,7 +28,8 @@ enum FileType { kCurrentFile, kTempFile, kInfoLogFile, // Either the current one, or an old one - kMetaDatabase + kMetaDatabase, + kIdentityFile }; // Return the name of the log file with the specified number @@ -82,6 +83,11 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, extern std::string MetaDatabaseName(const std::string& dbname, uint64_t number); +// Return the name of the Identity file which stores a unique number for the db +// that will get regenerated if the db loses all its data and is recreated fresh +// either from a backup-image or empty +extern std::string IdentityFileName(const std::string& dbname); + // If filename is a rocksdb file, store the type of the file in *type. // The number encoded in the filename is stored in *number. If the // filename was successfully parsed, returns true. Else return false. @@ -94,5 +100,7 @@ extern bool ParseFileName(const std::string& filename, extern Status SetCurrentFile(Env* env, const std::string& dbname, uint64_t descriptor_number); +// Make the IDENTITY file for the db +extern Status SetIdentityFile(Env* env, const std::string& dbname); } // namespace rocksdb diff --git a/include/rocksdb/env.h b/include/rocksdb/env.h index 3f9023c25..78ac83427 100644 --- a/include/rocksdb/env.h +++ b/include/rocksdb/env.h @@ -233,6 +233,9 @@ class Env { // Converts seconds-since-Jan-01-1970 to a printable string virtual std::string TimeToString(uint64_t time) = 0; + // Generates a unique id that can be used to identify a db + virtual std::string GenerateUniqueId(); + private: // No copying allowed Env(const Env&); diff --git a/util/env_posix.cc b/util/env_posix.cc index 4cb434b77..80141fd58 100644 --- a/util/env_posix.cc +++ b/util/env_posix.cc @@ -1386,6 +1386,25 @@ void PosixEnv::StartThread(void (*function)(void* arg), void* arg) { } // namespace +std::string Env::GenerateUniqueId() { + std::string uuid_file = "/proc/sys/kernel/random/uuid"; + if (FileExists(uuid_file)) { + std::string uuid; + Status s = ReadFileToString(this, uuid_file, &uuid); + if (s.ok()) { + return uuid; + } + } + // Could not read uuid_file - generate uuid using "nanos-random" + Random64 r(time(nullptr)); + uint64_t random_uuid_portion = + r.Uniform(std::numeric_limits::max()); + uint64_t nanos_uuid_portion = NowNanos(); + char uuid2[200]; + snprintf(uuid2, 200, "%lx-%lx", nanos_uuid_portion, random_uuid_portion); + return uuid2; +} + Env* Env::Default() { static PosixEnv default_env; return &default_env;