diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..a1e9a48e4 --- /dev/null +++ b/.clang-format @@ -0,0 +1,47 @@ +# Complete list of style options can be found at: +# http://clang.llvm.org/docs/ClangFormatStyleOptions.html +--- +BasedOnStyle: Google +AccessModifierOffset: -1 +ConstructorInitializerIndentWidth: 4 +AlignEscapedNewlinesLeft: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakTemplateDeclarations: true +AlwaysBreakBeforeMultilineStrings: true +BreakBeforeBinaryOperators: false +BreakConstructorInitializersBeforeComma: false +BinPackParameters: false +ColumnLimit: 80 +ConstructorInitializerAllOnOneLineOrOnePerLine: true +DerivePointerBinding: true +ExperimentalAutoDetectBinPacking: true +IndentCaseLabels: false +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 10 +PenaltyBreakComment: 60 +PenaltyBreakString: 1000 +PenaltyBreakFirstLessLess: 20 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerBindsToType: true +SpacesBeforeTrailingComments: 2 +Cpp11BracedListStyle: true +Standard: Cpp11 +IndentWidth: 2 +TabWidth: 8 +UseTab: Never +BreakBeforeBraces: Attach +IndentFunctionDeclarationAfterType: false +SpacesInParentheses: false +SpacesInAngles: false +SpaceInEmptyParentheses: false +SpacesInCStyleCastParentheses: false +SpaceAfterControlStatementKeyword: true +SpaceBeforeAssignmentOperators: true +ContinuationIndentWidth: 4 +... diff --git a/INSTALL.md b/INSTALL.md index 07d975068..ab0460341 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -44,5 +44,7 @@ libraries. You are on your own. `make clean; make` will compile librocksdb.a (RocskDB static library) and all the unit tests. You can run all unit tests with `make check`. +For shared library builds, exec `make librocksdb.so` instead. + If you followed the above steps and your compile or unit tests fail, please submit an issue: (https://github.com/facebook/rocksdb/issues) diff --git a/Makefile b/Makefile index 21cd4cfd2..1412489c2 100644 --- a/Makefile +++ b/Makefile @@ -45,8 +45,8 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind) VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full TESTS = \ - autovector_test \ db_test \ + autovector_test \ table_properties_collector_test \ arena_test \ auto_roll_logger_test \ diff --git a/README.fb b/README.fb index d17eed832..d3cc4110b 100644 --- a/README.fb +++ b/README.fb @@ -1,3 +1,3 @@ * Detailed instructions on how to compile using fbcode and jemalloc -* Latest release is 2.5.fb +* Latest release is 2.7.fb diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 96a1fb331..87c4c871d 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -198,7 +198,7 @@ EOF } EOF if [ "$?" = 0 ]; then - COMMON_FLAGS="$PLATFORM_LDFLAGS -DROCKSDB_FALLOCATE_PRESENT" + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT" fi # Test whether Snappy library is installed diff --git a/build_tools/build_detect_version b/build_tools/build_detect_version index eef905d8e..3ee6c92bd 100755 --- a/build_tools/build_detect_version +++ b/build_tools/build_detect_version @@ -8,7 +8,7 @@ # # create git version file -VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp +VFILE=$PWD/util/build_version.cc.tmp trap "rm $VFILE" EXIT # check to see if git is in the path @@ -36,7 +36,7 @@ echo "const char* rocksdb_build_git_datetime = \"rocksdb_build_git_datetime:$(da echo "const char* rocksdb_build_compile_date = __DATE__;" >> ${VFILE} echo "const char* rocksdb_build_compile_time = __TIME__;" >> ${VFILE} -OUTFILE=$ROCKSDB_ROOT/util/build_version.cc +OUTFILE=$PWD/util/build_version.cc if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then cp $VFILE $OUTFILE fi diff --git a/build_tools/fbcode.gcc471.sh b/build_tools/fbcode.gcc471.sh index e8a0cdeaa..929405715 100644 --- a/build_tools/fbcode.gcc471.sh +++ b/build_tools/fbcode.gcc471.sh @@ -54,7 +54,7 @@ RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic" CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC" CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE" -CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT" CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2" EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a" diff --git a/build_tools/fbcode.gcc481.sh b/build_tools/fbcode.gcc481.sh index 7ca337cf2..ae2bb57da 100644 --- a/build_tools/fbcode.gcc481.sh +++ b/build_tools/fbcode.gcc481.sh @@ -61,7 +61,7 @@ RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic" CFLAGS+=" -nostdlib $LIBGCC_INCLUDE $GLIBC_INCLUDE" -CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT" CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2" EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so" diff --git a/build_tools/make_new_version.sh b/build_tools/make_new_version.sh new file mode 100755 index 000000000..ca8a21262 --- /dev/null +++ b/build_tools/make_new_version.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Copyright (c) 2013, Facebook, Inc. All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +set -e +# Print out the colored progress info so that it can be brainlessly +# distinguished by users. +function title() { + echo -e "\033[1;32m$*\033[0m" +} + +usage="Create new rocksdb version and prepare it for the release process\n" +usage+="USAGE: ./make_new_version.sh " + +# -- Pre-check +if [[ $# < 1 ]]; then + echo -e $usage + exit 1 +fi + +ROCKSDB_VERSION=$1 + +GIT_BRANCH=`git rev-parse --abbrev-ref HEAD` +if [ $GIT_BRANCH != "master" ]; then + echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch." +fi + +# --Step 1: cutting new tag +title "Adding new tag for this release ..." +git tag -a "$ROCKSDB_VERSION.fb" -m "Rocksdb $ROCKSDB_VERSION" + +# Setting up the proxy for remote repo access +export http_proxy=http://172.31.255.99:8080 +export https_proxy="$http_proxy"; + +title "Pushing new tag to remote repo ..." +proxycmd.sh git push origin --tags + +# --Step 2: Update README.fb +title "Updating the latest version info in README.fb ..." +sed -i "s/Latest release is [0-9]\+.[0-9]\+.fb/Latest release is $ROCKSDB_VERSION.fb/" README.fb +git commit README.fb -m "update the latest version in README.fb to $ROCKSDB_VERSION" +proxycmd.sh git push + +# --Step 3: Prepare this repo for 3rd release +title "Cleaning up repo ..." +make clean +git clean -fxd + +title "Generating the build info ..." +# Comment out the call of `build_detection_version` so that the SHA number and build date of this +# release will remain constant. Otherwise everytime we run "make" util/build_version.cc will be +# overridden. +sed -i 's/^\$PWD\/build_tools\/build_detect_version$//' build_tools/build_detect_platform + +# Generate util/build_version.cc +build_tools/build_detect_version + +title "Done!" diff --git a/db/db_filesnapshot.cc b/db/db_filesnapshot.cc index 7b9c5ddeb..a7232246a 100644 --- a/db/db_filesnapshot.cc +++ b/db/db_filesnapshot.cc @@ -22,20 +22,34 @@ namespace rocksdb { Status DBImpl::DisableFileDeletions() { MutexLock l(&mutex_); - disable_delete_obsolete_files_ = true; - Log(options_.info_log, "File Deletions Disabled"); + ++disable_delete_obsolete_files_; + if (disable_delete_obsolete_files_ == 1) { + // if not, it has already been disabled, so don't log anything + Log(options_.info_log, "File Deletions Disabled"); + } return Status::OK(); } -Status DBImpl::EnableFileDeletions() { +Status DBImpl::EnableFileDeletions(bool force) { DeletionState deletion_state; + bool should_purge_files = false; { MutexLock l(&mutex_); - disable_delete_obsolete_files_ = false; - Log(options_.info_log, "File Deletions Enabled"); - FindObsoleteFiles(deletion_state, true); + if (force) { + // if force, we need to enable file deletions right away + disable_delete_obsolete_files_ = 0; + } else if (disable_delete_obsolete_files_ > 0) { + --disable_delete_obsolete_files_; + } + if (disable_delete_obsolete_files_ == 0) { + Log(options_.info_log, "File Deletions Enabled"); + should_purge_files = true; + FindObsoleteFiles(deletion_state, true); + } + } + if (should_purge_files) { + PurgeObsoleteFiles(deletion_state); } - PurgeObsoleteFiles(deletion_state); LogFlush(options_.info_log); return Status::OK(); } diff --git a/db/db_impl.cc b/db/db_impl.cc index d63f145b3..093112857 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -247,7 +247,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname) bg_logstats_scheduled_(false), manual_compaction_(nullptr), logger_(nullptr), - disable_delete_obsolete_files_(false), + disable_delete_obsolete_files_(0), delete_obsolete_files_last_run_(options.env->NowMicros()), purge_wal_files_last_run_(0), last_stats_dump_time_microsec_(0), @@ -509,7 +509,7 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, mutex_.AssertHeld(); // if deletion is disabled, do nothing - if (disable_delete_obsolete_files_) { + if (disable_delete_obsolete_files_ > 0) { return; } @@ -1262,7 +1262,7 @@ Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress, MaybeScheduleLogDBDeployStats(); - if (!disable_delete_obsolete_files_) { + if (disable_delete_obsolete_files_ == 0) { // add to deletion state deletion_state.log_delete_files.insert( deletion_state.log_delete_files.end(), diff --git a/db/db_impl.h b/db/db_impl.h index a11106f91..3b02ee9b1 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -73,7 +73,7 @@ class DBImpl : public DB { virtual const Options& GetOptions() const; virtual Status Flush(const FlushOptions& options); virtual Status DisableFileDeletions(); - virtual Status EnableFileDeletions(); + virtual Status EnableFileDeletions(bool force); // All the returned filenames start with "/" virtual Status GetLiveFiles(std::vector&, uint64_t* manifest_file_size, @@ -417,7 +417,12 @@ class DBImpl : public DB { int64_t volatile last_log_ts; // shall we disable deletion of obsolete files - bool disable_delete_obsolete_files_; + // if 0 the deletion is enabled. + // if non-zero, files will not be getting deleted + // This enables two different threads to call + // EnableFileDeletions() and DisableFileDeletions() + // without any synchronization + int disable_delete_obsolete_files_; // last time when DeleteObsoleteFiles was invoked uint64_t delete_obsolete_files_last_run_; diff --git a/db/db_impl_readonly.h b/db/db_impl_readonly.h index af9c79ed0..4beaedd01 100644 --- a/db/db_impl_readonly.h +++ b/db/db_impl_readonly.h @@ -55,7 +55,7 @@ public: virtual Status DisableFileDeletions() { return Status::NotSupported("Not supported operation in read only mode."); } - virtual Status EnableFileDeletions() { + virtual Status EnableFileDeletions(bool force) { return Status::NotSupported("Not supported operation in read only mode."); } virtual Status GetLiveFiles(std::vector&, diff --git a/db/db_test.cc b/db/db_test.cc index ff053c217..1e39454df 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -4381,7 +4381,7 @@ class ModelDB: public DB { virtual Status DisableFileDeletions() { return Status::OK(); } - virtual Status EnableFileDeletions() { + virtual Status EnableFileDeletions(bool force) { return Status::OK(); } virtual Status GetLiveFiles(std::vector&, uint64_t* size, diff --git a/db/table_cache.cc b/db/table_cache.cc index 736bf60f5..593352dde 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -29,6 +29,11 @@ static void UnrefEntry(void* arg1, void* arg2) { cache->Release(h); } +static Slice GetSliceForFileNumber(uint64_t* file_number) { + return Slice(reinterpret_cast(file_number), + sizeof(*file_number)); +} + TableCache::TableCache(const std::string& dbname, const Options* options, const EnvOptions& storage_options, @@ -50,8 +55,7 @@ Status TableCache::FindTable(const EnvOptions& toptions, Cache::Handle** handle, bool* table_io, const bool no_io) { Status s; - Slice key(reinterpret_cast(&file_number), sizeof(file_number)); - + Slice key = GetSliceForFileNumber(&file_number); *handle = cache_->Lookup(key); if (*handle == nullptr) { if (no_io) { // Dont do IO and return a not-found status @@ -164,8 +168,7 @@ bool TableCache::PrefixMayMatch(const ReadOptions& options, } void TableCache::Evict(uint64_t file_number) { - Slice key(reinterpret_cast(&file_number), sizeof(file_number)); - cache_->Erase(key); + cache_->Erase(GetSliceForFileNumber(&file_number)); } } // namespace rocksdb diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index c4c5aa87f..dd17d9e9b 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -247,7 +247,15 @@ class DB { virtual Status DisableFileDeletions() = 0; // Allow compactions to delete obselete files. - virtual Status EnableFileDeletions() = 0; + // If force == true, the call to EnableFileDeletions() will guarantee that + // file deletions are enabled after the call, even if DisableFileDeletions() + // was called multiple times before. + // If force == false, EnableFileDeletions will only enable file deletion + // after it's been called at least as many times as DisableFileDeletions(), + // enabling the two methods to be called by two threads concurrently without + // synchronization -- i.e., file deletions will be enabled only after both + // threads call EnableFileDeletions() + virtual Status EnableFileDeletions(bool force = true) = 0; // GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup diff --git a/include/utilities/stackable_db.h b/include/utilities/stackable_db.h index 2d86a611b..908fe10b7 100644 --- a/include/utilities/stackable_db.h +++ b/include/utilities/stackable_db.h @@ -123,8 +123,8 @@ class StackableDB : public DB { return db_->DisableFileDeletions(); } - virtual Status EnableFileDeletions() override { - return db_->EnableFileDeletions(); + virtual Status EnableFileDeletions(bool force) override { + return db_->EnableFileDeletions(force); } virtual Status GetLiveFiles(std::vector& vec, uint64_t* mfs, diff --git a/utilities/backupable/backupable_db_test.cc b/utilities/backupable/backupable_db_test.cc index af4af0d02..c64f0170b 100644 --- a/utilities/backupable/backupable_db_test.cc +++ b/utilities/backupable/backupable_db_test.cc @@ -48,7 +48,7 @@ class DummyDB : public StackableDB { return options_; } - virtual Status EnableFileDeletions() override { + virtual Status EnableFileDeletions(bool force) override { ASSERT_TRUE(!deletions_enabled_); deletions_enabled_ = true; return Status::OK();