Make builds reproducible (#7866)

Summary:
Closes https://github.com/facebook/rocksdb/issues/7035

Changed how build_version.cc was generated:
- Included the GIT tag/branch in the build_version file
- Changed the "Build Date" to be:
      - If the GIT branch is "clean" (no changes), the date of the last git commit
      - If the branch is not clean, the current date
 - Added APIs to access the "build information", rather than accessing the strings directly.

The build_version.cc file is now regenerated whenever the library objects are rebuilt.

Verified that the built files remain the same size across builds on a "clean build" and the same information is reported by sst_dump --version

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7866

Reviewed By: pdillinger

Differential Revision: D26086565

Pulled By: mrambacher

fbshipit-source-id: 6fcbe47f6033989d5cf26a0ccb6dfdd9dd239d7f
main
mrambacher 4 years ago committed by Facebook GitHub Bot
parent c696f27432
commit 0a9a05ae12
  1. 31
      CMakeLists.txt
  2. 1
      HISTORY.md
  3. 56
      Makefile
  4. 23
      db/db_impl/db_impl.cc
  5. 1
      include/rocksdb/options.h
  6. 25
      include/rocksdb/version.h
  7. 3
      tools/sst_dump_tool.cc
  8. 74
      util/build_version.cc.in
  9. 15
      util/build_version.h

@ -170,23 +170,25 @@ else()
endif() endif()
endif() endif()
string(TIMESTAMP TS "%Y/%m/%d %H:%M:%S" UTC) string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC)
set(GIT_DATE_TIME "${TS}" CACHE STRING "the time we first built rocksdb") set(BUILD_DATE "${TS}" CACHE STRING "the time we first built rocksdb")
find_package(Git) find_package(Git)
if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
if(WIN32) execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_SHA COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD )
execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA) execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" RESULT_VARIABLE GIT_MOD COMMAND "${GIT_EXECUTABLE}" diff-index HEAD --quiet)
else() execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_DATE COMMAND "${GIT_EXECUTABLE}" log -1 --date=format:"%Y-%m-%d %T" --format="%ad")
execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA) execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG RESULT_VARIABLE rv COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q --short HEAD OUTPUT_STRIP_TRAILING_WHITESPACE)
if (rv AND NOT rv EQUAL 0)
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG COMMAND "${GIT_EXECUTABLE}" describe --tags --exact-match OUTPUT_STRIP_TRAILING_WHITESPACE)
endif() endif()
else() else()
set(GIT_SHA 0) set(GIT_SHA 0)
set(GIT_MOD 0)
endif() endif()
string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}")
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}") string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}")
option(WITH_MD_LIBRARY "build with MD" ON) option(WITH_MD_LIBRARY "build with MD" ON)
if(WIN32 AND MSVC) if(WIN32 AND MSVC)
@ -199,9 +201,7 @@ endif()
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc) set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY) configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
add_library(build_version OBJECT ${BUILD_VERSION_CC})
target_include_directories(build_version PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/util)
if(MSVC) if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
@ -831,8 +831,7 @@ set(SOURCES
utilities/transactions/write_unprepared_txn_db.cc utilities/transactions/write_unprepared_txn_db.cc
utilities/ttl/db_ttl_impl.cc utilities/ttl/db_ttl_impl.cc
utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index.cc
utilities/write_batch_with_index/write_batch_with_index_internal.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc)
$<TARGET_OBJECTS:build_version>)
list(APPEND SOURCES list(APPEND SOURCES
utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc
@ -923,12 +922,12 @@ else()
set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT}) set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
endif() endif()
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES}) add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES} ${BUILD_VERSION_CC})
target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE
${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
if(ROCKSDB_BUILD_SHARED) if(ROCKSDB_BUILD_SHARED)
add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES}) add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES} ${BUILD_VERSION_CC})
target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE
${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})

@ -15,6 +15,7 @@
### Public API Change ### Public API Change
* Add a public API WriteBufferManager::dummy_entries_in_cache_usage() which reports the size of dummy entries stored in cache (passed to WriteBufferManager). Dummy entries are used to account for DataBlocks. * Add a public API WriteBufferManager::dummy_entries_in_cache_usage() which reports the size of dummy entries stored in cache (passed to WriteBufferManager). Dummy entries are used to account for DataBlocks.
* Add a SystemClock class that contains the time-related methods from Env. The original methods in Env may be deprecated in a future release. This class will allow easier testing, development, and expansion of time-related features. * Add a SystemClock class that contains the time-related methods from Env. The original methods in Env may be deprecated in a future release. This class will allow easier testing, development, and expansion of time-related features.
* Add a public API GetRocksBuildProperties and GetRocksBuildInfoAsString to get properties about the current build. These properties may include settings related to the GIT settings (branch, timestamp). This change also sets the "build date" based on the GIT properties, rather than the actual build time, thereby enabling more reproducible builds.
## 6.16.0 (12/18/2020) ## 6.16.0 (12/18/2020)
### Behavior Changes ### Behavior Changes
* Attempting to write a merge operand without explicitly configuring `merge_operator` now fails immediately, causing the DB to enter read-only mode. Previously, failure was deferred until the `merge_operator` was needed by a user read or a background operation. * Attempting to write a merge operand without explicitly configuring `merge_operator` now fails immediately, causing the DB to enter read-only mode. Previously, failure was deferred until the `merge_operator` was needed by a user read or a background operation.

@ -479,31 +479,6 @@ CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverl
LDFLAGS += $(PLATFORM_LDFLAGS) LDFLAGS += $(PLATFORM_LDFLAGS)
# If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but
# the file needs to already exist or else the build will fail
ifndef NO_UPDATE_BUILD_VERSION
date := $(shell date +%F)
ifdef FORCE_GIT_SHA
git_sha := $(FORCE_GIT_SHA)
else
git_sha := $(shell git rev-parse HEAD 2>/dev/null)
endif
gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ util/build_version.cc.in
# Record the version of the source that we are compiling.
# We keep a record of the git revision in this file. It is then built
# as a regular source file as part of the compilation process.
# One can run "strings executable_filename | grep _build_" to find
# the version of the source that we used to build the executable file.
FORCE:
util/build_version.cc: FORCE
$(AM_V_GEN)rm -f $@-t
$(AM_V_at)$(gen_build_version) > $@-t
$(AM_V_at)if test -f $@; then \
cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@; \
else mv -f $@-t $@; fi
endif
OBJ_DIR?=. OBJ_DIR?=.
LIB_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(LIB_SOURCES)) LIB_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(LIB_SOURCES))
ifeq ($(HAVE_POWER8),1) ifeq ($(HAVE_POWER8),1)
@ -863,6 +838,37 @@ ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h |
ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3) ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
ROCKSDB_PATCH = $(shell egrep "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3) ROCKSDB_PATCH = $(shell egrep "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
# If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but
# the file needs to already exist or else the build will fail
ifndef NO_UPDATE_BUILD_VERSION
# By default, use the current date-time as the date. If there are no changes,
# we will use the last commit date instead.
build_date := $(shell date "+%Y-%m-%d %T")
ifdef FORCE_GIT_SHA
git_sha := $(FORCE_GIT_SHA)
git_mod := 1
git_date := $(build_date)
else
git_sha := $(shell git rev-parse HEAD 2>/dev/null)
git_tag := $(shell git symbolic-ref -q --short HEAD || git describe --tags --exact-match 2>/dev/null)
git_mod := $(shell git diff-index HEAD --quiet 2>/dev/null; echo $$?)
git_date := $(shell git log -1 --date=format:"%Y-%m-%d %T" --format="%ad" 2>/dev/null)
endif
gen_build_version = sed -e s/@GIT_SHA@/$(git_sha)/ -e s:@GIT_TAG@:"$(git_tag)": -e s/@GIT_MOD@/"$(git_mod)"/ -e s/@BUILD_DATE@/"$(build_date)"/ -e s/@GIT_DATE@/"$(git_date)"/ util/build_version.cc.in
# Record the version of the source that we are compiling.
# We keep a record of the git revision in this file. It is then built
# as a regular source file as part of the compilation process.
# One can run "strings executable_filename | grep _build_" to find
# the version of the source that we used to build the executable file.
util/build_version.cc: $(filter-out $(OBJ_DIR)/util/build_version.o, $(LIB_OBJECTS)) util/build_version.cc.in
$(AM_V_GEN)rm -f $@-t
$(AM_V_at)$(gen_build_version) > $@
endif
CLEAN_FILES += util/build_version.cc
default: all default: all
#----------------------------------------------- #-----------------------------------------------

@ -82,6 +82,7 @@
#include "rocksdb/stats_history.h" #include "rocksdb/stats_history.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/version.h"
#include "rocksdb/write_buffer_manager.h" #include "rocksdb/write_buffer_manager.h"
#include "table/block_based/block.h" #include "table/block_based/block.h"
#include "table/block_based/block_based_table_factory.h" #include "table/block_based/block_based_table_factory.h"
@ -93,7 +94,6 @@
#include "table/two_level_iterator.h" #include "table/two_level_iterator.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "util/autovector.h" #include "util/autovector.h"
#include "util/build_version.h"
#include "util/cast_util.h" #include "util/cast_util.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/compression.h" #include "util/compression.h"
@ -4207,16 +4207,17 @@ void DBImpl::EraseThreadStatusDbInfo() const {}
// //
// A global method that can dump out the build version // A global method that can dump out the build version
void DumpRocksDBBuildVersion(Logger* log) { void DumpRocksDBBuildVersion(Logger* log) {
#if !defined(IOS_CROSS_COMPILE) ROCKS_LOG_HEADER(log, "RocksDB version: %s\n",
// if we compile with Xcode, we don't run build_detect_version, so we don't GetRocksVersionAsString().c_str());
// generate util/build_version.cc const auto& props = GetRocksBuildProperties();
ROCKS_LOG_HEADER(log, "RocksDB version: %d.%d.%d\n", ROCKSDB_MAJOR, const auto& sha = props.find("rocksdb_build_git_sha");
ROCKSDB_MINOR, ROCKSDB_PATCH); if (sha != props.end()) {
ROCKS_LOG_HEADER(log, "Git sha %s", rocksdb_build_git_sha); ROCKS_LOG_HEADER(log, "Git sha %s", sha->second.c_str());
ROCKS_LOG_HEADER(log, "Compile date %s", rocksdb_build_compile_date); }
#else const auto date = props.find("rocksdb_build_date");
(void)log; // ignore "-Wunused-parameter" if (date != props.end()) {
#endif ROCKS_LOG_HEADER(log, "Compile date %s", date->second.c_str());
}
} }
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE

@ -26,7 +26,6 @@
#include "rocksdb/sst_partitioner.h" #include "rocksdb/sst_partitioner.h"
#include "rocksdb/types.h" #include "rocksdb/types.h"
#include "rocksdb/universal_compaction.h" #include "rocksdb/universal_compaction.h"
#include "rocksdb/version.h"
#include "rocksdb/write_buffer_manager.h" #include "rocksdb/write_buffer_manager.h"
#ifdef max #ifdef max

@ -4,6 +4,11 @@
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#pragma once #pragma once
#include <string>
#include <unordered_map>
#include "rocksdb/rocksdb_namespace.h"
#define ROCKSDB_MAJOR 6 #define ROCKSDB_MAJOR 6
#define ROCKSDB_MINOR 17 #define ROCKSDB_MINOR 17
#define ROCKSDB_PATCH 0 #define ROCKSDB_PATCH 0
@ -14,3 +19,23 @@
#define __ROCKSDB_MAJOR__ ROCKSDB_MAJOR #define __ROCKSDB_MAJOR__ ROCKSDB_MAJOR
#define __ROCKSDB_MINOR__ ROCKSDB_MINOR #define __ROCKSDB_MINOR__ ROCKSDB_MINOR
#define __ROCKSDB_PATCH__ ROCKSDB_PATCH #define __ROCKSDB_PATCH__ ROCKSDB_PATCH
namespace ROCKSDB_NAMESPACE {
// Returns a set of properties indicating how/when/where this version of RocksDB
// was created.
const std::unordered_map<std::string, std::string>& GetRocksBuildProperties();
// Returns the current version of RocksDB as a string (e.g. "6.16.0").
// If with_patch is true, the patch is included (6.16.x).
// Otherwise, only major and minor version is included (6.16)
std::string GetRocksVersionAsString(bool with_patch = true);
// Gets the set of build properties (@see GetRocksBuildProperties) into a
// string. Properties are returned one-per-line, with the first line being:
// "<program> from RocksDB <version>.
// If verbose is true, the full set of properties is
// printed. If verbose is false, only the version information (@see
// GetRocksVersionString) is printed.
std::string GetRocksBuildInfoAsString(const std::string& program,
bool verbose = false);
} // namespace ROCKSDB_NAMESPACE

@ -280,8 +280,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
print_help(/*to_stderr*/ false); print_help(/*to_stderr*/ false);
return 0; return 0;
} else if (strcmp(argv[i], "--version") == 0) { } else if (strcmp(argv[i], "--version") == 0) {
printf("sst_dump from RocksDB %d.%d.%d\n", ROCKSDB_MAJOR, ROCKSDB_MINOR, printf("%s\n", GetRocksBuildInfoAsString("sst_dump").c_str());
ROCKSDB_PATCH);
return 0; return 0;
} else { } else {
fprintf(stderr, "Unrecognized argument '%s'\n\n", argv[i]); fprintf(stderr, "Unrecognized argument '%s'\n\n", argv[i]);

@ -1,5 +1,71 @@
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#include "build_version.h"
const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:@@GIT_SHA@@"; #include <memory>
const char* rocksdb_build_git_date = "rocksdb_build_git_date:@@GIT_DATE_TIME@@";
const char* rocksdb_build_compile_date = __DATE__; #include "rocksdb/version.h"
#include "util/string_util.h"
// The build script may replace these values with real values based
// on whether or not GIT is available and the platform settings
static const std::string rocksdb_build_git_sha = "rocksdb_build_git_sha:@GIT_SHA@";
static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:@GIT_TAG@";
#define HAS_GIT_CHANGES @GIT_MOD@
#if HAS_GIT_CHANGES == 0
// If HAS_GIT_CHANGES is 0, the GIT date is used.
// Use the time the branch/tag was last modified
static const std::string rocksdb_build_date = "rocksdb_build_date:@GIT_DATE@";
#else
// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
// Use the time the build was created.
static const std::string rocksdb_build_date = "rocksdb_build_date:@BUILD_DATE@";
#endif
namespace ROCKSDB_NAMESPACE {
static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
size_t colon = name.find(":");
if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) {
// If we found a "@:", then this property was a build-time substitution that failed. Skip it
size_t at = name.find("@", colon);
if (at != colon + 1) {
// Everything before the colon is the name, after is the value
(*props)[name.substr(0, colon)] = name.substr(colon + 1);
}
}
}
static std::unordered_map<std::string, std::string>* LoadPropertiesSet() {
auto * properties = new std::unordered_map<std::string, std::string>();
AddProperty(properties, rocksdb_build_git_sha);
AddProperty(properties, rocksdb_build_git_tag);
AddProperty(properties, rocksdb_build_date);
return properties;
}
const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
static std::unique_ptr<std::unordered_map<std::string, std::string>> props(LoadPropertiesSet());
return *props;
}
std::string GetRocksVersionAsString(bool with_patch) {
std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR);
if (with_patch) {
return version + "." + ToString(ROCKSDB_PATCH);
} else {
return version;
}
}
std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {
std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true);
if (verbose) {
for (const auto& it : GetRocksBuildProperties()) {
info.append("\n ");
info.append(it.first);
info.append(": ");
info.append(it.second);
}
}
return info;
}
} // namespace ROCKSDB_NAMESPACE

@ -1,15 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#pragma once
#if !defined(IOS_CROSS_COMPILE)
// if we compile with Xcode, we don't run build_detect_version, so we don't
// generate these variables
// this variable tells us about the git revision
extern const char* rocksdb_build_git_sha;
// Date on which the code was compiled:
extern const char* rocksdb_build_compile_date;
#endif
Loading…
Cancel
Save